OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge with morphLBM to incorporate the newest backend updates

Browse Source
This commit is contained in:
Rex Zhe Li 2020-02-04 13:28:57 -05:00
commit 72ab9f803e
145 changed files with 11487 additions and 5848 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

344
CMakeLists.txt
View File

@ -1,170 +1,174 @@
# Set some CMake properties
CMAKE_MINIMUM_REQUIRED( VERSION 3.9 )
MESSAGE("====================")
MESSAGE("Configuring LBPM-WIA")
MESSAGE("====================")
# Set the project name
SET( PROJ LBPM ) # Set the project name for CMake
SET( LBPM_LIB lbpm-wia ) # Set the final library name
SET( LBPM_INC ) # Set an optional subfolder for includes (e.g. include/name/...)
SET( TEST_MAX_PROCS 16 )
# Initialize the project
PROJECT( ${PROJ} LANGUAGES CXX )
# Prevent users from building in place
IF ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" )
MESSAGE( FATAL_ERROR "Building code in place is a bad idea" )
ENDIF()
# Set the default C++ standard
SET( CMAKE_CXX_EXTENSIONS OFF )
IF ( NOT CMAKE_CXX_STANDARD )
IF ( CXX_STD )
MESSAGE( FATAL_ERROR "CXX_STD is obsolete, please set CMAKE_CXX_STANDARD" )
ENDIF()
SET( CMAKE_CXX_STANDARD 14 )
ENDIF()
IF ( ( "${CMAKE_CXX_STANDARD}" GREATER "90" ) OR ( "${CMAKE_CXX_STANDARD}" LESS "14" ) )
MESSAGE( FATAL_ERROR "C++14 or newer required" )
ENDIF()
# Set source/install paths
SET( ${PROJ}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" )
SET( ${PROJ}_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
IF( ${PROJ}_INSTALL_DIR )
SET( ${PROJ}_INSTALL_DIR "${${PROJ}_INSTALL_DIR}" )
ELSEIF( PREFIX )
SET( ${PROJ}_INSTALL_DIR "${PREFIX}" )
ELSEIF( NOT ${PROJ}_INSTALL_DIR )
SET( ${PROJ}_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
ENDIF()
INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
SET( CMAKE_MODULE_PATH ${${PROJ}_SOURCE_DIR} ${${PROJ}_SOURCE_DIR}/cmake )
# Include macros
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake" )
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libraries.cmake" )
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LBPM-macros.cmake" )
# Check if we are only compiling docs
CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 )
# Set testing paramaters
SET( DROP_METHOD "http" )
SET( DROP_SITE "" )
SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
SET( TRIGGER_SITE "" )
SET( DROP_SITE_CDASH TRUE )
ENABLE_TESTING()
INCLUDE( CTest )
# Check the compile mode and compile flags
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_SYSTEM()
ENDIF()
# Add some directories to include
INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
# Create the target for documentation
ADD_CUSTOM_TARGET( doc )
ADD_CUSTOM_TARGET( latex_docs )
CHECK_ENABLE_FLAG( USE_DOXYGEN 1 )
CHECK_ENABLE_FLAG( USE_LATEX 1 )
FILE( MAKE_DIRECTORY "${${PROJ}_INSTALL_DIR}/doc" )
IF ( USE_DOXYGEN )
SET( DOXYFILE_LATEX YES )
SET( DOXYFILE_IN "${${PROJ}_SOURCE_DIR}/doxygen/Doxyfile.in" )
SET( DOXY_HEADER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/header.html" )
SET( DOXY_FOOTER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/footer.html" )
SET( DOXYFILE_OUTPUT_DIR "${${PROJ}_INSTALL_DIR}/doc" )
SET( DOXYFILE_SRC_HTML_DIR "${${PROJ}_SOURCE_DIR}/doxygen/html" )
SET( DOXYFILE_SOURCE_DIR "${${PROJ}_SOURCE_DIR}" )
SET( REL_PACKAGE_HTML "" )
SET( DOXYGEN_MACROS "" )
MESSAGE("DOXYGEN_MACROS = ${DOXYGEN_MACROS}")
INCLUDE( "${${PROJ}_SOURCE_DIR}/cmake/UseDoxygen.cmake" )
IF ( DOXYGEN_FOUND )
ADD_DEPENDENCIES( doxygen latex_docs )
ADD_DEPENDENCIES( doc latex_docs doxygen )
ELSE()
SET( USE_DOXYGEN 0 )
ENDIF()
ENDIF()
# Create custom targets for build-test, check, and distclean
ADD_CUSTOM_TARGET( build-test )
ADD_CUSTOM_TARGET( build-examples )
ADD_CUSTOM_TARGET( check COMMAND make test )
ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common IO threadpool StackTrace )
# Check for CUDA
CHECK_ENABLE_FLAG( USE_CUDA 0 )
NULL_USE( CMAKE_CUDA_FLAGS )
IF ( USE_CUDA )
ADD_DEFINITIONS( -DUSE_CUDA )
ENABLE_LANGUAGE( CUDA )
ENDIF()
# Configure external packages
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_MPI() # MPI must be before other libraries
CONFIGURE_MIC()
CONFIGURE_NETCDF()
CONFIGURE_SILO()
CONFIGURE_LBPM()
CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" )
CONFIGURE_LINE_COVERAGE()
# Set the external library link list
SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )
ENDIF()
# Macro to create 1,2,4 processor tests
MACRO( ADD_LBPM_TEST_1_2_4 EXENAME ${ARGN} )
ADD_LBPM_TEST( ${EXENAME} ${ARGN} )
ADD_LBPM_TEST_PARALLEL( ${EXENAME} 2 ${ARGN} )
ADD_LBPM_TEST_PARALLEL( ${EXENAME} 4 ${ARGN} )
ENDMACRO()
# Add the src directories
IF ( NOT ONLY_BUILD_DOCS )
BEGIN_PACKAGE_CONFIG( lbpm-wia-library )
ADD_PACKAGE_SUBDIRECTORY( common )
ADD_PACKAGE_SUBDIRECTORY( analysis )
ADD_PACKAGE_SUBDIRECTORY( IO )
ADD_PACKAGE_SUBDIRECTORY( threadpool )
ADD_PACKAGE_SUBDIRECTORY( StackTrace )
ADD_PACKAGE_SUBDIRECTORY( models )
IF ( USE_CUDA )
ADD_PACKAGE_SUBDIRECTORY( gpu )
ELSE()
ADD_PACKAGE_SUBDIRECTORY( cpu )
ENDIF()
INSTALL_LBPM_TARGET( lbpm-wia-library )
ADD_SUBDIRECTORY( tests )
ADD_SUBDIRECTORY( example )
#ADD_SUBDIRECTORY( workflows )
INSTALL_PROJ_LIB()
ENDIF()
# Set some CMake properties
CMAKE_MINIMUM_REQUIRED( VERSION 3.9 )
MESSAGE("====================")
MESSAGE("Configuring LBPM-WIA")
MESSAGE("====================")
# Set the project name
SET( PROJ LBPM ) # Set the project name for CMake
SET( LBPM_LIB lbpm-wia ) # Set the final library name
SET( LBPM_INC ) # Set an optional subfolder for includes (e.g. include/name/...)
SET( TEST_MAX_PROCS 16 )
# Initialize the project
PROJECT( ${PROJ} LANGUAGES CXX )
# Prevent users from building in place
IF ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" )
MESSAGE( FATAL_ERROR "Building code in place is a bad idea" )
ENDIF()
# Set the default C++ standard
SET( CMAKE_CXX_EXTENSIONS OFF )
IF ( NOT CMAKE_CXX_STANDARD )
IF ( CXX_STD )
MESSAGE( FATAL_ERROR "CXX_STD is obsolete, please set CMAKE_CXX_STANDARD" )
ENDIF()
SET( CMAKE_CXX_STANDARD 14 )
ENDIF()
IF ( ( "${CMAKE_CXX_STANDARD}" GREATER "90" ) OR ( "${CMAKE_CXX_STANDARD}" LESS "14" ) )
MESSAGE( FATAL_ERROR "C++14 or newer required" )
ENDIF()
# Set source/install paths
SET( ${PROJ}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" )
SET( ${PROJ}_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
IF( ${PROJ}_INSTALL_DIR )
SET( ${PROJ}_INSTALL_DIR "${${PROJ}_INSTALL_DIR}" )
ELSEIF( PREFIX )
SET( ${PROJ}_INSTALL_DIR "${PREFIX}" )
ELSEIF( NOT ${PROJ}_INSTALL_DIR )
SET( ${PROJ}_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
ENDIF()
INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
SET( CMAKE_MODULE_PATH ${${PROJ}_SOURCE_DIR} ${${PROJ}_SOURCE_DIR}/cmake )
# Include macros
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake" )
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libraries.cmake" )
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LBPM-macros.cmake" )
# Check if we are only compiling docs
CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 )
# Set testing paramaters
SET( DROP_METHOD "http" )
SET( DROP_SITE "" )
SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
SET( TRIGGER_SITE "" )
SET( DROP_SITE_CDASH TRUE )
ENABLE_TESTING()
INCLUDE( CTest )
# Check the compile mode and compile flags
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_SYSTEM()
ENDIF()
# Add some directories to include
INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
# Create the target for documentation
ADD_CUSTOM_TARGET( doc )
ADD_CUSTOM_TARGET( latex_docs )
CHECK_ENABLE_FLAG( USE_DOXYGEN 1 )
CHECK_ENABLE_FLAG( USE_LATEX 1 )
FILE( MAKE_DIRECTORY "${${PROJ}_INSTALL_DIR}/doc" )
IF ( USE_DOXYGEN )
SET( DOXYFILE_LATEX YES )
SET( DOXYFILE_IN "${${PROJ}_SOURCE_DIR}/doxygen/Doxyfile.in" )
SET( DOXY_HEADER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/header.html" )
SET( DOXY_FOOTER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/footer.html" )
SET( DOXYFILE_OUTPUT_DIR "${${PROJ}_INSTALL_DIR}/doc" )
SET( DOXYFILE_SRC_HTML_DIR "${${PROJ}_SOURCE_DIR}/doxygen/html" )
SET( DOXYFILE_SOURCE_DIR "${${PROJ}_SOURCE_DIR}" )
SET( REL_PACKAGE_HTML "" )
SET( DOXYGEN_MACROS "" )
MESSAGE("DOXYGEN_MACROS = ${DOXYGEN_MACROS}")
INCLUDE( "${${PROJ}_SOURCE_DIR}/cmake/UseDoxygen.cmake" )
IF ( DOXYGEN_FOUND )
ADD_DEPENDENCIES( doxygen latex_docs )
ADD_DEPENDENCIES( doc latex_docs doxygen )
ELSE()
SET( USE_DOXYGEN 0 )
ENDIF()
ENDIF()
# Create custom targets for build-test, check, and distclean
ADD_CUSTOM_TARGET( build-test )
ADD_CUSTOM_TARGET( build-examples )
ADD_CUSTOM_TARGET( check COMMAND make test )
ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common IO threadpool StackTrace )
# Check for CUDA
CHECK_ENABLE_FLAG( USE_CUDA 0 )
CHECK_ENABLE_FLAG( USE_HIP 0 )
NULL_USE( CMAKE_CUDA_FLAGS )
IF ( USE_CUDA )
ADD_DEFINITIONS( -DUSE_CUDA )
ENABLE_LANGUAGE( CUDA )
ELSEIF ( USE_HIP )
FIND_PACKAGE( HIP )
MESSAGE( FATAL_ERROR "STOP" )
ENDIF()
# Configure external packages
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_MPI() # MPI must be before other libraries
CONFIGURE_MIC()
CONFIGURE_NETCDF()
CONFIGURE_SILO()
CONFIGURE_LBPM()
CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" )
CONFIGURE_LINE_COVERAGE()
# Set the external library link list
SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )
ENDIF()
# Macro to create 1,2,4 processor tests
MACRO( ADD_LBPM_TEST_1_2_4 EXENAME ${ARGN} )
ADD_LBPM_TEST( ${EXENAME} ${ARGN} )
ADD_LBPM_TEST_PARALLEL( ${EXENAME} 2 ${ARGN} )
ADD_LBPM_TEST_PARALLEL( ${EXENAME} 4 ${ARGN} )
ENDMACRO()
# Add the src directories
IF ( NOT ONLY_BUILD_DOCS )
BEGIN_PACKAGE_CONFIG( lbpm-wia-library )
ADD_PACKAGE_SUBDIRECTORY( common )
ADD_PACKAGE_SUBDIRECTORY( analysis )
ADD_PACKAGE_SUBDIRECTORY( IO )
ADD_PACKAGE_SUBDIRECTORY( threadpool )
ADD_PACKAGE_SUBDIRECTORY( StackTrace )
ADD_PACKAGE_SUBDIRECTORY( models )
IF ( USE_CUDA )
ADD_PACKAGE_SUBDIRECTORY( gpu )
ELSE()
ADD_PACKAGE_SUBDIRECTORY( cpu )
ENDIF()
INSTALL_LBPM_TARGET( lbpm-wia-library )
ADD_SUBDIRECTORY( tests )
ADD_SUBDIRECTORY( example )
#ADD_SUBDIRECTORY( workflows )
INSTALL_PROJ_LIB()
ENDIF()

145
IO/MeshDatabase.cpp
View File

@ -1,7 +1,8 @@
#include "IO/MeshDatabase.h"
#include "IO/Mesh.h"
#include "IO/PackData.h"
#include "IO/IOHelpers.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <vector>
@ -13,8 +14,6 @@
/****************************************************
****************************************************/
// MeshType
template<>
size_t packsize<IO::MeshType>( const IO::MeshType& rhs )
@ -247,80 +246,76 @@ void DatabaseEntry::read( const std::string& line )
// Gather the mesh databases from all processors
inline int tod( int N ) { return (N+7)/sizeof(double); }
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, MPI_Comm comm )
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, const Utilities::MPI& comm )
{
#ifdef USE_MPI
PROFILE_START("gatherAll");
PROFILE_START("gatherAll-pack",2);
int size = MPI_WORLD_SIZE();
// First pack the mesh data to local buffers
int localsize = 0;
for (size_t i=0; i<meshes.size(); i++)
localsize += tod(packsize(meshes[i]));
auto localbuf = new double[localsize];
int pos = 0;
for (size_t i=0; i<meshes.size(); i++) {
pack( meshes[i], (char*) &localbuf[pos] );
pos += tod(packsize(meshes[i]));
}
PROFILE_STOP("gatherAll-pack",2);
// Get the number of bytes each processor will be sending/recieving
PROFILE_START("gatherAll-send1",2);
auto recvsize = new int[size];
MPI_Allgather(&localsize,1,MPI_INT,recvsize,1,MPI_INT,comm);
int globalsize = recvsize[0];
auto disp = new int[size];
disp[0] = 0;
for (int i=1; i<size; i++) {
disp[i] = disp[i-1] + recvsize[i];
globalsize += recvsize[i];
}
PROFILE_STOP("gatherAll-send1",2);
// Send/recv the global data
PROFILE_START("gatherAll-send2",2);
auto globalbuf = new double[globalsize];
MPI_Allgatherv(localbuf,localsize,MPI_DOUBLE,globalbuf,recvsize,disp,MPI_DOUBLE,comm);
PROFILE_STOP("gatherAll-send2",2);
// Unpack the data
PROFILE_START("gatherAll-unpack",2);
std::map<std::string,MeshDatabase> data;
pos = 0;
while ( pos < globalsize ) {
MeshDatabase tmp;
unpack(tmp,(char*)&globalbuf[pos]);
pos += tod(packsize(tmp));
std::map<std::string,MeshDatabase>::iterator it = data.find(tmp.name);
if ( it==data.end() ) {
data[tmp.name] = tmp;
} else {
for (size_t i=0; i<tmp.domains.size(); i++)
it->second.domains.push_back(tmp.domains[i]);
for (size_t i=0; i<tmp.variables.size(); i++)
it->second.variables.push_back(tmp.variables[i]);
it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end());
}
}
for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it) {
// Get the unique variables
std::set<VariableDatabase> data2(it->second.variables.begin(),it->second.variables.end());
it->second.variables = std::vector<VariableDatabase>(data2.begin(),data2.end());
}
// Free temporary memory
delete [] localbuf;
delete [] recvsize;
delete [] disp;
delete [] globalbuf;
// Return the results
std::vector<MeshDatabase> data2(data.size());
size_t i=0;
for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it, ++i)
data2[i] = it->second;
PROFILE_STOP("gatherAll-unpack",2);
PROFILE_STOP("gatherAll");
return data2;
#else
if ( comm.getSize() == 1 )
return meshes;
#endif
PROFILE_START("gatherAll");
PROFILE_START("gatherAll-pack",2);
int size = comm.getSize();
// First pack the mesh data to local buffers
int localsize = 0;
for (size_t i=0; i<meshes.size(); i++)
localsize += tod(packsize(meshes[i]));
auto localbuf = new double[localsize];
int pos = 0;
for (size_t i=0; i<meshes.size(); i++) {
pack( meshes[i], (char*) &localbuf[pos] );
pos += tod(packsize(meshes[i]));
}
PROFILE_STOP("gatherAll-pack",2);
// Get the number of bytes each processor will be sending/recieving
PROFILE_START("gatherAll-send1",2);
auto recvsize = comm.allGather( localsize );
int globalsize = recvsize[0];
auto disp = new int[size];
disp[0] = 0;
for (int i=1; i<size; i++) {
disp[i] = disp[i-1] + recvsize[i];
globalsize += recvsize[i];
}
PROFILE_STOP("gatherAll-send1",2);
// Send/recv the global data
PROFILE_START("gatherAll-send2",2);
auto globalbuf = new double[globalsize];
comm.allGather(localbuf,localsize,globalbuf,recvsize.data(),disp,true);
PROFILE_STOP("gatherAll-send2",2);
// Unpack the data
PROFILE_START("gatherAll-unpack",2);
std::map<std::string,MeshDatabase> data;
pos = 0;
while ( pos < globalsize ) {
MeshDatabase tmp;
unpack(tmp,(char*)&globalbuf[pos]);
pos += tod(packsize(tmp));
std::map<std::string,MeshDatabase>::iterator it = data.find(tmp.name);
if ( it==data.end() ) {
data[tmp.name] = tmp;
} else {
for (size_t i=0; i<tmp.domains.size(); i++)
it->second.domains.push_back(tmp.domains[i]);
for (size_t i=0; i<tmp.variables.size(); i++)
it->second.variables.push_back(tmp.variables[i]);
it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end());
}
}
for (auto it=data.begin(); it!=data.end(); ++it) {
// Get the unique variables
std::set<VariableDatabase> data2(it->second.variables.begin(),it->second.variables.end());
it->second.variables = std::vector<VariableDatabase>(data2.begin(),data2.end());
}
// Free temporary memory
delete [] localbuf;
delete [] disp;
delete [] globalbuf;
// Return the results
std::vector<MeshDatabase> data2(data.size());
size_t i=0;
for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it, ++i)
data2[i] = it->second;
PROFILE_STOP("gatherAll-unpack",2);
PROFILE_STOP("gatherAll");
return data2;
}

4
IO/MeshDatabase.h
View File

@ -2,7 +2,7 @@
#define MeshDatabase_INC
#include "IO/Mesh.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include <iostream>
#include <memory>
@ -70,7 +70,7 @@ public:
//! Gather the mesh databases from all processors
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, MPI_Comm comm );
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, const Utilities::MPI& comm );
//! Write the mesh databases to a file

12
IO/PIO.cpp
View File

@ -1,6 +1,6 @@
#include "IO/PIO.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include <fstream>
#include <string>
@ -36,10 +36,7 @@ static void shutdownFilestream( )
}
void Utilities::logOnlyNodeZero( const std::string &filename )
{
int rank = 0;
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
#endif
int rank = ::Utilities::MPI( MPI_COMM_WORLD ).getRank();
if ( rank == 0 )
logAllNodes(filename,true);
}
@ -54,10 +51,7 @@ void Utilities::logAllNodes( const std::string &filename, bool singleStream )
// Open the log stream and redirect output
std::string full_filename = filename;
if ( !singleStream ) {
int rank = 0;
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
#endif
int rank = ::Utilities::MPI( MPI_COMM_WORLD ).getRank();
char tmp[100];
sprintf(tmp,".%04i",rank);
full_filename += std::string(tmp);

105
IO/PackData.cpp Normal file
View File

@ -0,0 +1,105 @@
#include "IO/PackData.h"
#include <string.h>
/********************************************************
* Concrete implimentations for packing/unpacking *
********************************************************/
// unsigned char
template<>
size_t packsize<unsigned char>( const unsigned char& rhs )
{
return sizeof(unsigned char);
}
template<>
void pack<unsigned char>( const unsigned char& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(unsigned char));
}
template<>
void unpack<unsigned char>( unsigned char& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(unsigned char));
}
// char
template<>
size_t packsize<char>( const char& rhs )
{
return sizeof(char);
}
template<>
void pack<char>( const char& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(char));
}
template<>
void unpack<char>( char& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(char));
}
// int
template<>
size_t packsize<int>( const int& rhs )
{
return sizeof(int);
}
template<>
void pack<int>( const int& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(int));
}
template<>
void unpack<int>( int& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(int));
}
// unsigned int
template<>
size_t packsize<unsigned int>( const unsigned int& rhs )
{
return sizeof(unsigned int);
}
template<>
void pack<unsigned int>( const unsigned int& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(int));
}
template<>
void unpack<unsigned int>( unsigned int& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(int));
}
// size_t
template<>
size_t packsize<size_t>( const size_t& rhs )
{
return sizeof(size_t);
}
template<>
void pack<size_t>( const size_t& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(size_t));
}
template<>
void unpack<size_t>( size_t& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(size_t));
}
// std::string
template<>
size_t packsize<std::string>( const std::string& rhs )
{
return rhs.size()+1;
}
template<>
void pack<std::string>( const std::string& rhs, char *buffer )
{
memcpy(buffer,rhs.c_str(),rhs.size()+1);
}
template<>
void unpack<std::string>( std::string& data, const char *buffer )
{
data = std::string(buffer);
}

78
IO/PackData.h Normal file
View File

@ -0,0 +1,78 @@
// This file contains unctions to pack/unpack data structures
#ifndef included_PackData
#define included_PackData
#include <vector>
#include <set>
#include <map>
//! Template function to return the buffer size required to pack a class
template<class TYPE>
size_t packsize( const TYPE& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const TYPE& rhs, char *buffer );
//! Template function to unpack a class from a buffer
template<class TYPE>
void unpack( TYPE& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::vector
template<class TYPE>
size_t packsize( const std::vector<TYPE>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::vector<TYPE>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::vector<TYPE>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::pair
template<class TYPE1, class TYPE2>
size_t packsize( const std::pair<TYPE1,TYPE2>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::map
template<class TYPE1, class TYPE2>
size_t packsize( const std::map<TYPE1,TYPE2>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::set
template<class TYPE>
size_t packsize( const std::set<TYPE>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::set<TYPE>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::set<TYPE>& data, const char *buffer );
#include "IO/PackData.hpp"
#endif

9
common/MPI_Helpers.hpp → IO/PackData.hpp
View File

@ -1,8 +1,9 @@
// This file contains wrappers for MPI routines and functions to pack/unpack data structures
#ifndef MPI_WRAPPERS_HPP
#define MPI_WRAPPERS_HPP
// This file functions to pack/unpack data structures
#ifndef included_PackData_hpp
#define included_PackData_hpp
#include "IO/PackData.h"
#include "common/MPI_Helpers.h"
#include <string.h>
#include <vector>
#include <set>

20
IO/Writer.cpp
View File

@ -2,7 +2,7 @@
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "IO/silo.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <sys/stat.h>
@ -36,7 +36,7 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap
global_IO_format = Format::SILO;
else
ERROR("Unknown format");
int rank = comm_rank(MPI_COMM_WORLD);
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
if ( !append && rank==0 ) {
mkdir(path.c_str(),S_IRWXU|S_IRGRP);
std::string filename;
@ -55,7 +55,7 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap
// Write the mesh data in the original format
static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO::MeshDataStruct>& meshData, const std::string& path )
{
int rank = MPI_WORLD_RANK();
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
for (size_t i=0; i<meshData.size(); i++) {
char domainname[100], filename[100], fullpath[200];
@ -120,7 +120,7 @@ static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO
// Create the database entry for the mesh data
static IO::MeshDatabase getDatabase( const std::string& filename, const IO::MeshDataStruct& mesh, int format )
{
int rank = MPI_WORLD_RANK();
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
char domainname[100];
sprintf(domainname,"%s_%05i",mesh.meshName.c_str(),rank);
// Create the MeshDatabase
@ -161,7 +161,7 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
const IO::MeshDataStruct& mesh, int format )
{
const int level = 0;
int rank = MPI_WORLD_RANK();
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
// Create the MeshDatabase
IO::MeshDatabase database = getDatabase( filename, mesh, format );
// Write the mesh
@ -399,7 +399,7 @@ void writeSiloSummary( const std::vector<IO::MeshDatabase>& meshes_written, cons
static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
{
int rank = MPI_WORLD_RANK();
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
char filename[100], fullpath[200];
sprintf(filename,"%05i",rank);
@ -419,7 +419,7 @@ static std::vector<IO::MeshDatabase> writeMeshesSilo(
const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
{
#ifdef USE_SILO
int rank = MPI_WORLD_RANK();
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
char filename[100], fullpath[200];
sprintf(filename,"%05i.silo",rank);
@ -441,12 +441,12 @@ static std::vector<IO::MeshDatabase> writeMeshesSilo(
/****************************************************
* Write the mesh data *
****************************************************/
void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm )
void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm )
{
if ( global_IO_path.empty() )
IO::initialize( );
PROFILE_START("writeData");
int rank = comm_rank(comm);
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
// Check the meshData before writing
for ( const auto& data : meshData ) {
if ( !data.check() )
@ -457,7 +457,7 @@ void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStr
if ( rank == 0 ) {
mkdir(path.c_str(),S_IRWXU|S_IRGRP);
}
MPI_Barrier(comm);
comm.barrier();
// Write the mesh files
std::vector<IO::MeshDatabase> meshes_written;
if ( global_IO_format == Format::OLD ) {

4
IO/Writer.h
View File

@ -34,7 +34,7 @@ void initialize( const std::string& path="", const std::string& format="silo", b
* @param[in] meshData The data to write
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
*/
void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm );
void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm );
/*!
@ -44,7 +44,7 @@ void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>
* @param[in] meshData The data to write
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
*/
inline void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm )
inline void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm )
{
char subdir[100];
sprintf(subdir,"vis%03i",timestep);

15
IO/netcdf.cpp
View File

@ -1,6 +1,6 @@
#include "IO/netcdf.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "ProfilerApp.h"
@ -116,10 +116,10 @@ std::string VariableTypeName( VariableType type )
/****************************************************
* Open/close a file *
****************************************************/
int open( const std::string& filename, FileMode mode, MPI_Comm comm )
int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm )
{
int fid = 0;
if ( comm == MPI_COMM_NULL ) {
if ( comm.isNull() ) {
if ( mode == READ ) {
int err = nc_open( filename.c_str(), NC_NOWRITE, &fid );
CHECK_NC_ERR( err );
@ -134,13 +134,13 @@ int open( const std::string& filename, FileMode mode, MPI_Comm comm )
}
} else {
if ( mode == READ ) {
int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm, MPI_INFO_NULL, &fid );
int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else if ( mode == WRITE ) {
int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm, MPI_INFO_NULL, &fid );
int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else if ( mode == CREATE ) {
int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm, MPI_INFO_NULL, &fid );
int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else {
ERROR("Unknown file mode");
@ -189,6 +189,7 @@ std::vector<size_t> getAttDim( int fid, const std::string& att )
{
std::vector<size_t> dim(1,0);
int err = nc_inq_attlen( fid, NC_GLOBAL, att.c_str(), dim.data() );
CHECK_NC_ERR( err );
return dim;
}
std::vector<std::string> getVarNames( int fid )
@ -374,7 +375,7 @@ Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& sta
std::vector<size_t> var_size = getVarDim( fid, var );
for (int d=0; d<(int)var_size.size(); d++) {
if ( start[d]<0 || start[d]+stride[d]*(count[d]-1)>(int)var_size[d] ) {
int rank = comm_rank(MPI_COMM_WORLD);
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
char tmp[1000];
sprintf(tmp,"%i: Range exceeded array dimension:\n"
" start[%i]=%i, count[%i]=%i, stride[%i]=%i, var_size[%i]=%i",

4
IO/netcdf.h
View File

@ -5,7 +5,7 @@
#include <vector>
#include "common/Array.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
@ -32,7 +32,7 @@ std::string VariableTypeName( VariableType type );
* @param mode Open the file for reading or writing
* @param comm MPI communicator to use (MPI_COMM_WORLD: don't use parallel netcdf)
*/
int open( const std::string& filename, FileMode mode, MPI_Comm comm=MPI_COMM_NULL );
int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm=MPI_COMM_NULL );
/*!

2
IO/silo.cpp
View File

@ -1,6 +1,6 @@
#include "IO/silo.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "ProfilerApp.h"

2
IO/silo.h
View File

@ -6,7 +6,7 @@
#include <array>
#include "common/Array.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"

2
IO/silo.hpp
View File

@ -3,7 +3,7 @@
#include "IO/silo.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "ProfilerApp.h"

2
StackTrace/ErrorHandlers.h
View File

@ -6,7 +6,7 @@
#include <functional>
#include "mpi.h"
#include "common/MPI.h"
namespace StackTrace

26
StackTrace/StackTrace.cpp
View File

@ -7,6 +7,7 @@
#include <algorithm>
#include <atomic>
#include <cerrno>
#include <csignal>
#include <cstring>
#include <iostream>
@ -348,8 +349,11 @@ static inline int exec3( const char *cmd, FUNCTION &fun )
if ( buffer[0] != 0 )
fun( buffer );
}
auto status = pclose( pipe );
int code = WEXITSTATUS( status );
int code = pclose( pipe );
if ( errno == ECHILD ) {
errno = 0;
code = 0;
}
std::this_thread::yield(); // Allow any signals to process
resetSignal( SIGCHLD ); // Clear child exited
return code;
@ -1741,7 +1745,7 @@ std::vector<int> StackTrace::defaultSignalsToCatch()
* Set the signal handlers *
****************************************************************************/
static std::function<void( const StackTrace::abort_error &err )> abort_fun;
static StackTrace::abort_error rethrow()
StackTrace::abort_error rethrow()
{
StackTrace::abort_error error;
#ifdef USE_LINUX
@ -1775,14 +1779,14 @@ static StackTrace::abort_error rethrow()
}
return error;
}
static void term_func_abort( int sig )
void StackTrace::terminateFunctionSignal( int sig )
{
StackTrace::abort_error err;
err.type = StackTrace::terminateType::signal;
err.signal = sig;
err.bytes = StackTrace::Utilities::getMemoryUsage();
err.stack = StackTrace::backtrace();
err.stackType = StackTrace::printStackType::global;
err.stackType = StackTrace::getDefaultStackType();
abort_fun( err );
}
static bool signals_set[256] = { false };
@ -1829,7 +1833,7 @@ void StackTrace::setErrorHandler( std::function<void( const StackTrace::abort_er
{
abort_fun = abort;
std::set_terminate( term_func );
setSignals( defaultSignalsToCatch(), &term_func_abort );
setSignals( defaultSignalsToCatch(), &terminateFunctionSignal );
std::set_unexpected( term_func );
}
void StackTrace::clearErrorHandler()
@ -2215,7 +2219,7 @@ void StackTrace::cleanupStackTrace( multi_stack_info &stack )
// Remove callstack (and all children) for threads that are just contributing
bool test = function.find( "_callstack_signal_handler" ) != npos ||
function.find( "getGlobalCallStacks" ) != npos ||
function.find( "(" ) == npos;
function.find( "backtrace" ) != npos || function.find( "(" ) == npos;
if ( test ) {
it = stack.children.erase( it );
continue;
@ -2515,3 +2519,11 @@ const char *StackTrace::abort_error::what() const noexcept
d_msg.erase( i, 1 );
return d_msg.c_str();
}
/****************************************************************************
* Get/Set default stack type *
****************************************************************************/
static StackTrace::printStackType abort_stackType = StackTrace::printStackType::global;
void StackTrace::setDefaultStackType( StackTrace::printStackType type ) { abort_stackType = type; }
StackTrace::printStackType StackTrace::getDefaultStackType() { return abort_stackType; }

11
StackTrace/StackTrace.h
View File

@ -246,6 +246,10 @@ void clearSignals();
void raiseSignal( int signal );
//! Default function to abort after catching a signal
void terminateFunctionSignal( int signal );
//! Return a list of all signals that can be caught
std::vector<int> allSignalsToCatch();
@ -289,6 +293,13 @@ multi_stack_info generateFromString( const std::vector<std::string> &str );
multi_stack_info generateFromString( const std::string &str );
//! Set default stack type
void setDefaultStackType( StackTrace::printStackType );
//! Get default stack type
StackTrace::printStackType getDefaultStackType();
} // namespace StackTrace

60
StackTrace/Utilities.cpp
View File

@ -8,17 +8,23 @@
#include <cstring>
#include <fstream>
#include <iostream>
#include <mutex>
#include <sstream>
#include <stdexcept>
#include <typeinfo>
#ifdef USE_MPI
#include "mpi.h"
#include "common/MPI.h"
#endif
#ifdef USE_TIMER
#include "MemoryApp.h"
#endif
#ifdef USE_GCOV
extern "C" void __gcov_flush( void );
#endif
#define perr std::cerr
@ -65,6 +71,12 @@
// clang-format on
#ifdef __GNUC__
#define USE_ABI
#include <cxxabi.h>
#endif
namespace StackTrace {
@ -96,13 +108,12 @@ inline size_t findfirst( const std::vector<TYPE> &X, TYPE Y )
/****************************************************************************
* Function to terminate the program *
****************************************************************************/
static bool abort_throwException = false;
static printStackType abort_stackType = printStackType::global;
static int force_exit = 0;
static bool abort_throwException = false;
static int force_exit = 0;
void Utilities::setAbortBehavior( bool throwException, int stackType )
{
abort_throwException = throwException;
abort_stackType = static_cast<printStackType>( stackType );
StackTrace::setDefaultStackType( static_cast<printStackType>( stackType ) );
}
void Utilities::abort( const std::string &message, const std::string &filename, const int line )
{
@ -112,16 +123,28 @@ void Utilities::abort( const std::string &message, const std::string &filename,
err.type = terminateType::abort;
err.line = line;
err.bytes = Utilities::getMemoryUsage();
err.stackType = abort_stackType;
err.stackType = StackTrace::getDefaultStackType();
err.stack = StackTrace::backtrace();
throw err;
}
static void terminate( const StackTrace::abort_error &err )
static std::mutex terminate_mutex;
static inline void callAbort()
{
#ifdef USE_GCOV
__gcov_flush();
#endif
terminate_mutex.unlock();
std::abort();
}
void Utilities::terminate( const StackTrace::abort_error &err )
{
// Lock mutex to ensure multiple threads do not try to abort simultaneously
terminate_mutex.lock();
// Clear the error handlers
clearErrorHandler();
// Print the message and abort
if ( force_exit > 1 ) {
std::abort();
callAbort();
} else if ( !abort_throwException ) {
// Use MPI_abort (will terminate all processes)
force_exit = 2;
@ -135,10 +158,11 @@ static void terminate( const StackTrace::abort_error &err )
MPI_Abort( MPI_COMM_WORLD, -1 );
}
#endif
std::abort();
callAbort();
} else {
perr << err.what();
std::abort();
perr.flush();
callAbort();
}
}
@ -149,7 +173,7 @@ static void terminate( const StackTrace::abort_error &err )
static void setTerminateErrorHandler()
{
// Set the terminate routine for runtime errors
StackTrace::setErrorHandler( terminate );
StackTrace::setErrorHandler( Utilities::terminate );
}
void Utilities::setErrorHandlers()
{
@ -293,4 +317,18 @@ std::string Utilities::exec( const string_view &cmd, int &exit_code )
}
/****************************************************************************
* Get the type name *
****************************************************************************/
std::string Utilities::getTypeName( const std::type_info &id )
{
std::string name = id.name();
#if defined( USE_ABI )
int status;
name = abi::__cxa_demangle( name.c_str(), 0, 0, &status );
#endif
return name;
}
} // namespace StackTrace

18
StackTrace/Utilities.h
View File

@ -4,6 +4,7 @@
#include <stdexcept>
#include <string>
#include <thread>
#include <typeinfo>
#include "StackTrace/StackTrace.h"
#include "StackTrace/string_view.h"
@ -28,9 +29,14 @@ void abort( const std::string &message, const std::string &filename, const int l
void setAbortBehavior( bool throwException, int stackType = 2 );
//! Function to terminate the application
void terminate( const StackTrace::abort_error &err );
//! Function to set the error handlers
void setErrorHandlers();
//! Function to clear the error handlers
void clearErrorHandlers();
@ -92,6 +98,18 @@ void cause_segfault();
std::string exec( const StackTrace::string_view &cmd, int &exit_code );
//! Return the hopefully demangled name of the given type
std::string getTypeName( const std::type_info &id );
//! Return the hopefully demangled name of the given type
template<class TYPE>
inline std::string getTypeName()
{
return getTypeName( typeid( TYPE ) );
}
} // namespace Utilities
} // namespace StackTrace

2
StackTrace/string_view.h
View File

@ -119,7 +119,7 @@ public:
int result = 0;
for ( int i = 0; i < N && result == 0; i++ )
if ( d_data[i] != other[i] )
result = d_data[i] < other[i] ? -i : i;
result = d_data[i] < other[i] ? -( i + 1 ) : ( i + 1 );
if ( result == 0 )
result = size() == other.size() ? 0 : size() < other.size() ? -1 : 1;
return result;

16
analysis/Minkowski.cpp
View File

@ -4,7 +4,7 @@
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
@ -109,13 +109,13 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue)
// convert X for 2D manifold to 3D object
Xi *= 0.5;
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
// Phase averages
MPI_Allreduce(&Vi,&Vi_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Xi,&Xi_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Ai,&Ai_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Ji,&Ji_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Barrier(Dm->Comm);
Vi_global = Dm->Comm.sumReduce( Vi );
Xi_global = Dm->Comm.sumReduce( Xi );
Ai_global = Dm->Comm.sumReduce( Ai );
Ji_global = Dm->Comm.sumReduce( Ji );
Dm->Comm.barrier();
PROFILE_STOP("ComputeScalar");
}
@ -168,7 +168,7 @@ int Minkowski::MeasureConnectedPathway(){
double vF=0.0;
n_connected_components = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,Dm->rank_info,distance,distance,vF,vF,label,Dm->Comm);
// int n_connected_components = ComputeGlobalPhaseComponent(Nx-2,Ny-2,Nz-2,Dm->rank_info,const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, Dm->Comm )
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){

2
analysis/Minkowski.h
View File

@ -12,7 +12,7 @@
#include "analysis/distance.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"

171
analysis/SubPhase.cpp
View File

@ -93,7 +93,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
{
// If timelog is empty, write a short header to list the averages
//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
fprintf(TIMELOG,"sw krw krn qw qn pw pn\n");
fprintf(TIMELOG,"sw krw krn vw vn pw pn\n");
}
}
}
@ -160,9 +160,6 @@ void SubPhase::Basic(){
// If external boundary conditions are set, do not average over the inlet
kmin=1; kmax=Nz-1;
if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4;
if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
imin=jmin=1;
// If inlet/outlet layers exist use these as default
if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x;
@ -172,7 +169,6 @@ void SubPhase::Basic(){
nb.reset(); wb.reset();
double nA,nB;
double count_w = 0.0;
double count_n = 0.0;
@ -233,25 +229,25 @@ void SubPhase::Basic(){
}
}
}
gwb.V=sumReduce( Dm->Comm, wb.V);
gnb.V=sumReduce( Dm->Comm, nb.V);
gwb.M=sumReduce( Dm->Comm, wb.M);
gnb.M=sumReduce( Dm->Comm, nb.M);
gwb.Px=sumReduce( Dm->Comm, wb.Px);
gwb.Py=sumReduce( Dm->Comm, wb.Py);
gwb.Pz=sumReduce( Dm->Comm, wb.Pz);
gnb.Px=sumReduce( Dm->Comm, nb.Px);
gnb.Py=sumReduce( Dm->Comm, nb.Py);
gnb.Pz=sumReduce( Dm->Comm, nb.Pz);
gwb.V = Dm->Comm.sumReduce( wb.V);
gnb.V = Dm->Comm.sumReduce( nb.V);
gwb.M = Dm->Comm.sumReduce( wb.M);
gnb.M = Dm->Comm.sumReduce( nb.M);
gwb.Px = Dm->Comm.sumReduce( wb.Px);
gwb.Py = Dm->Comm.sumReduce( wb.Py);
gwb.Pz = Dm->Comm.sumReduce( wb.Pz);
gnb.Px = Dm->Comm.sumReduce( nb.Px);
gnb.Py = Dm->Comm.sumReduce( nb.Py);
gnb.Pz = Dm->Comm.sumReduce( nb.Pz);
count_w=sumReduce( Dm->Comm, count_w);
count_n=sumReduce( Dm->Comm, count_n);
count_w = Dm->Comm.sumReduce( count_w);
count_n = Dm->Comm.sumReduce( count_n);
if (count_w > 0.0)
gwb.p=sumReduce( Dm->Comm, wb.p) / count_w;
gwb.p = Dm->Comm.sumReduce(wb.p) / count_w;
else
gwb.p = 0.0;
if (count_n > 0.0)
gnb.p=sumReduce( Dm->Comm, nb.p) / count_n;
gnb.p = Dm->Comm.sumReduce( nb.p) / count_n;
else
gnb.p = 0.0;
@ -300,14 +296,14 @@ void SubPhase::Basic(){
double saturation=gwb.V/(gwb.V + gnb.V);
double water_flow_rate=gwb.V*(gwb.Px*dir_x + gwb.Py*dir_y + gwb.Pz*dir_z)/gwb.M / Dm->Volume;
double not_water_flow_rate=gnb.V*(gnb.Px*dir_x + gnb.Py*dir_y + gnb.Pz*dir_z)/gnb.M/ Dm->Volume;
double total_flow_rate = water_flow_rate + not_water_flow_rate;
double fractional_flow= water_flow_rate / total_flow_rate;
//double total_flow_rate = water_flow_rate + not_water_flow_rate;
//double fractional_flow = water_flow_rate / total_flow_rate;
double h = Dm->voxel_length;
double krn = h*h*nu_n*not_water_flow_rate / force_mag ;
double krw = h*h*nu_w*water_flow_rate / force_mag;
//printf(" water saturation = %f, fractional flow =%f \n",saturation,fractional_flow);
fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*h*h*water_flow_rate,h*h*h*not_water_flow_rate, gwb.p, gnb.p);
fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p);
fflush(TIMELOG);
}
if (err==true){
@ -448,14 +444,14 @@ void SubPhase::Full(){
nd.X -= nc.X;
// compute global entities
gnc.V=sumReduce( Dm->Comm, nc.V);
gnc.A=sumReduce( Dm->Comm, nc.A);
gnc.H=sumReduce( Dm->Comm, nc.H);
gnc.X=sumReduce( Dm->Comm, nc.X);
gnd.V=sumReduce( Dm->Comm, nd.V);
gnd.A=sumReduce( Dm->Comm, nd.A);
gnd.H=sumReduce( Dm->Comm, nd.H);
gnd.X=sumReduce( Dm->Comm, nd.X);
gnc.V = Dm->Comm.sumReduce( nc.V );
gnc.A = Dm->Comm.sumReduce( nc.A );
gnc.H = Dm->Comm.sumReduce( nc.H );
gnc.X = Dm->Comm.sumReduce( nc.X );
gnd.V = Dm->Comm.sumReduce( nd.V );
gnd.A = Dm->Comm.sumReduce( nd.A );
gnd.H = Dm->Comm.sumReduce( nd.H );
gnd.X = Dm->Comm.sumReduce( nd.X );
gnd.Nc = nd.Nc;
// wetting
for (k=0; k<Nz; k++){
@ -495,14 +491,14 @@ void SubPhase::Full(){
wd.H -= wc.H;
wd.X -= wc.X;
// compute global entities
gwc.V=sumReduce( Dm->Comm, wc.V);
gwc.A=sumReduce( Dm->Comm, wc.A);
gwc.H=sumReduce( Dm->Comm, wc.H);
gwc.X=sumReduce( Dm->Comm, wc.X);
gwd.V=sumReduce( Dm->Comm, wd.V);
gwd.A=sumReduce( Dm->Comm, wd.A);
gwd.H=sumReduce( Dm->Comm, wd.H);
gwd.X=sumReduce( Dm->Comm, wd.X);
gwc.V = Dm->Comm.sumReduce( wc.V );
gwc.A = Dm->Comm.sumReduce( wc.A );
gwc.H = Dm->Comm.sumReduce( wc.H );
gwc.X = Dm->Comm.sumReduce( wc.X );
gwd.V = Dm->Comm.sumReduce( wd.V );
gwd.A = Dm->Comm.sumReduce( wd.A );
gwd.H = Dm->Comm.sumReduce( wd.H );
gwd.X = Dm->Comm.sumReduce( wd.X );
gwd.Nc = wd.Nc;
/* Set up geometric analysis of interface region */
@ -530,20 +526,20 @@ void SubPhase::Full(){
iwn.A = morph_i->A();
iwn.H = morph_i->H();
iwn.X = morph_i->X();
giwn.V=sumReduce( Dm->Comm, iwn.V);
giwn.A=sumReduce( Dm->Comm, iwn.A);
giwn.H=sumReduce( Dm->Comm, iwn.H);
giwn.X=sumReduce( Dm->Comm, iwn.X);
giwn.V = Dm->Comm.sumReduce( iwn.V );
giwn.A = Dm->Comm.sumReduce( iwn.A );
giwn.H = Dm->Comm.sumReduce( iwn.H );
giwn.X = Dm->Comm.sumReduce( iwn.X );
// measure only the connected part
iwnc.Nc = morph_i->MeasureConnectedPathway();
iwnc.V = morph_i->V();
iwnc.A = morph_i->A();
iwnc.H = morph_i->H();
iwnc.X = morph_i->X();
giwnc.V=sumReduce( Dm->Comm, iwnc.V);
giwnc.A=sumReduce( Dm->Comm, iwnc.A);
giwnc.H=sumReduce( Dm->Comm, iwnc.H);
giwnc.X=sumReduce( Dm->Comm, iwnc.X);
giwnc.V = Dm->Comm.sumReduce( iwnc.V );
giwnc.A = Dm->Comm.sumReduce( iwnc.A );
giwnc.H = Dm->Comm.sumReduce( iwnc.H );
giwnc.X = Dm->Comm.sumReduce( iwnc.X );
giwnc.Nc = iwnc.Nc;
double vol_nc_bulk = 0.0;
@ -634,46 +630,46 @@ void SubPhase::Full(){
}
}
gnd.M=sumReduce( Dm->Comm, nd.M);
gnd.Px=sumReduce( Dm->Comm, nd.Px);
gnd.Py=sumReduce( Dm->Comm, nd.Py);
gnd.Pz=sumReduce( Dm->Comm, nd.Pz);
gnd.K=sumReduce( Dm->Comm, nd.K);
gnd.M = Dm->Comm.sumReduce( nd.M );
gnd.Px = Dm->Comm.sumReduce( nd.Px );
gnd.Py = Dm->Comm.sumReduce( nd.Py );
gnd.Pz = Dm->Comm.sumReduce( nd.Pz );
gnd.K = Dm->Comm.sumReduce( nd.K );
gwd.M=sumReduce( Dm->Comm, wd.M);
gwd.Px=sumReduce( Dm->Comm, wd.Px);
gwd.Py=sumReduce( Dm->Comm, wd.Py);
gwd.Pz=sumReduce( Dm->Comm, wd.Pz);
gwd.K=sumReduce( Dm->Comm, wd.K);
gwd.M = Dm->Comm.sumReduce( wd.M );
gwd.Px = Dm->Comm.sumReduce( wd.Px );
gwd.Py = Dm->Comm.sumReduce( wd.Py );
gwd.Pz = Dm->Comm.sumReduce( wd.Pz );
gwd.K = Dm->Comm.sumReduce( wd.K );
gnc.M=sumReduce( Dm->Comm, nc.M);
gnc.Px=sumReduce( Dm->Comm, nc.Px);
gnc.Py=sumReduce( Dm->Comm, nc.Py);
gnc.Pz=sumReduce( Dm->Comm, nc.Pz);
gnc.K=sumReduce( Dm->Comm, nc.K);
gnc.M = Dm->Comm.sumReduce( nc.M );
gnc.Px = Dm->Comm.sumReduce( nc.Px );
gnc.Py = Dm->Comm.sumReduce( nc.Py );
gnc.Pz = Dm->Comm.sumReduce( nc.Pz );
gnc.K = Dm->Comm.sumReduce( nc.K );
gwc.M=sumReduce( Dm->Comm, wc.M);
gwc.Px=sumReduce( Dm->Comm, wc.Px);
gwc.Py=sumReduce( Dm->Comm, wc.Py);
gwc.Pz=sumReduce( Dm->Comm, wc.Pz);
gwc.K=sumReduce( Dm->Comm, wc.K);
gwc.M = Dm->Comm.sumReduce( wc.M );
gwc.Px = Dm->Comm.sumReduce( wc.Px );
gwc.Py = Dm->Comm.sumReduce( wc.Py );
gwc.Pz = Dm->Comm.sumReduce( wc.Pz );
gwc.K = Dm->Comm.sumReduce( wc.K );
giwn.Mn=sumReduce( Dm->Comm, iwn.Mn);
giwn.Pnx=sumReduce( Dm->Comm, iwn.Pnx);
giwn.Pny=sumReduce( Dm->Comm, iwn.Pny);
giwn.Pnz=sumReduce( Dm->Comm, iwn.Pnz);
giwn.Kn=sumReduce( Dm->Comm, iwn.Kn);
giwn.Mw=sumReduce( Dm->Comm, iwn.Mw);
giwn.Pwx=sumReduce( Dm->Comm, iwn.Pwx);
giwn.Pwy=sumReduce( Dm->Comm, iwn.Pwy);
giwn.Pwz=sumReduce( Dm->Comm, iwn.Pwz);
giwn.Kw=sumReduce( Dm->Comm, iwn.Kw);
giwn.Mn = Dm->Comm.sumReduce( iwn.Mn );
giwn.Pnx = Dm->Comm.sumReduce( iwn.Pnx );
giwn.Pny = Dm->Comm.sumReduce( iwn.Pny );
giwn.Pnz = Dm->Comm.sumReduce( iwn.Pnz );
giwn.Kn = Dm->Comm.sumReduce( iwn.Kn );
giwn.Mw = Dm->Comm.sumReduce( iwn.Mw );
giwn.Pwx = Dm->Comm.sumReduce( iwn.Pwx );
giwn.Pwy = Dm->Comm.sumReduce( iwn.Pwy );
giwn.Pwz = Dm->Comm.sumReduce( iwn.Pwz );
giwn.Kw = Dm->Comm.sumReduce( iwn.Kw );
// pressure averaging
gnc.p=sumReduce( Dm->Comm, nc.p);
gnd.p=sumReduce( Dm->Comm, nd.p);
gwc.p=sumReduce( Dm->Comm, wc.p);
gwd.p=sumReduce( Dm->Comm, wd.p);
gnc.p = Dm->Comm.sumReduce( nc.p );
gnd.p = Dm->Comm.sumReduce( nd.p );
gwc.p = Dm->Comm.sumReduce( wc.p );
gwd.p = Dm->Comm.sumReduce( wd.p );
if (vol_wc_bulk > 0.0)
wc.p = wc.p /vol_wc_bulk;
@ -684,10 +680,10 @@ void SubPhase::Full(){
if (vol_nd_bulk > 0.0)
nd.p = nd.p /vol_nd_bulk;
vol_wc_bulk=sumReduce( Dm->Comm, vol_wc_bulk);
vol_wd_bulk=sumReduce( Dm->Comm, vol_wd_bulk);
vol_nc_bulk=sumReduce( Dm->Comm, vol_nc_bulk);
vol_nd_bulk=sumReduce( Dm->Comm, vol_nd_bulk);
vol_wc_bulk = Dm->Comm.sumReduce( vol_wc_bulk );
vol_wd_bulk = Dm->Comm.sumReduce( vol_wd_bulk );
vol_nc_bulk = Dm->Comm.sumReduce( vol_nc_bulk );
vol_nd_bulk = Dm->Comm.sumReduce( vol_nd_bulk );
if (vol_wc_bulk > 0.0)
gwc.p = gwc.p /vol_wc_bulk;
@ -700,7 +696,8 @@ void SubPhase::Full(){
}
void SubPhase::AggregateLabels(char *FILENAME){
void SubPhase::AggregateLabels( const std::string& filename )
{
int nx = Dm->Nx;
int ny = Dm->Ny;
@ -722,9 +719,9 @@ void SubPhase::AggregateLabels(char *FILENAME){
}
}
}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
Dm->AggregateLabels(FILENAME);
Dm->AggregateLabels( filename );
}

4
analysis/SubPhase.h
View File

@ -12,7 +12,7 @@
#include "analysis/distance.h"
#include "analysis/Minkowski.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
@ -101,7 +101,7 @@ public:
void Basic();
void Full();
void Write(int time);
void AggregateLabels(char *FILENAME);
void AggregateLabels( const std::string& filename );
private:
FILE *TIMELOG;

103
analysis/TwoPhase.cpp
View File

@ -5,7 +5,7 @@
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
@ -204,6 +204,7 @@ TwoPhase::~TwoPhase()
void TwoPhase::ColorToSignedDistance(double Beta, DoubleArray &ColorData, DoubleArray &DistData)
{
NULL_USE( Beta );
/*double factor,temp,value;
factor=0.5/Beta;
// Initialize to -1,1 (segmentation)
@ -627,8 +628,8 @@ void TwoPhase::ComputeLocal()
void TwoPhase::AssignComponentLabels()
{
int LabelNWP=1;
int LabelWP=2;
//int LabelNWP=1;
//int LabelWP=2;
// NOTE: labeling the wetting phase components is tricky! One sandstone media had over 800,000 components
// NumberComponents_WP = ComputeGlobalPhaseComponent(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,PhaseID,LabelWP,Label_WP);
// treat all wetting phase is connected
@ -881,7 +882,7 @@ void TwoPhase::ComponentAverages()
}
}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
if (Dm->rank()==0){
printf("Component averages computed locally -- reducing result... \n");
}
@ -889,14 +890,14 @@ void TwoPhase::ComponentAverages()
RecvBuffer.resize(BLOB_AVG_COUNT,NumberComponents_NWP);
/* for (int b=0; b<NumberComponents_NWP; b++){
MPI_Barrier(Dm->Comm);
MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm);
Dm->Comm.barrier();
Dm->Comm.sumReduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT);
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_NWP(idx,b)=RecvBuffer(idx);
}
*/
MPI_Barrier(Dm->Comm);
MPI_Allreduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP, MPI_DOUBLE,MPI_SUM,Dm->Comm);
// MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm);
Dm->Comm.barrier();
Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP);
// Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT);
if (Dm->rank()==0){
printf("rescaling... \n");
@ -992,9 +993,8 @@ void TwoPhase::ComponentAverages()
// reduce the wetting phase averages
for (int b=0; b<NumberComponents_WP; b++){
MPI_Barrier(Dm->Comm);
// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Reduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm);
Dm->Comm.barrier();
Dm->Comm.sumReduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT);
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_WP(idx,b)=RecvBuffer(idx);
}
@ -1077,43 +1077,48 @@ void TwoPhase::Reduce()
int i;
double iVol_global=1.0/Volume;
//...........................................................................
MPI_Barrier(Dm->Comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wp_volume,&wp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&KGwns,&KGwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&KNwns,&KNwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wwndnw,&wwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wwnsdnwn,&wwnsdnwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jwnwwndnw,&Jwnwwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
Dm->Comm.barrier();
nwp_volume_global = Dm->Comm.sumReduce( nwp_volume );
wp_volume_global = Dm->Comm.sumReduce( wp_volume );
awn_global = Dm->Comm.sumReduce( awn );
ans_global = Dm->Comm.sumReduce( ans );
aws_global = Dm->Comm.sumReduce( aws );
lwns_global = Dm->Comm.sumReduce( lwns );
As_global = Dm->Comm.sumReduce( As );
Jwn_global = Dm->Comm.sumReduce( Jwn );
Kwn_global = Dm->Comm.sumReduce( Kwn );
KGwns_global = Dm->Comm.sumReduce( KGwns );
KNwns_global = Dm->Comm.sumReduce( KNwns );
efawns_global = Dm->Comm.sumReduce( efawns );
wwndnw_global = Dm->Comm.sumReduce( wwndnw );
wwnsdnwn_global = Dm->Comm.sumReduce( wwnsdnwn );
Jwnwwndnw_global = Dm->Comm.sumReduce( Jwnwwndnw );
// Phase averages
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vawns(0),&vawns_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&euler,&euler_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&An,&An_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jn,&Jn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Kn,&Kn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Barrier(Dm->Comm);
vol_w_global = Dm->Comm.sumReduce( vol_w );
vol_n_global = Dm->Comm.sumReduce( vol_n );
paw_global = Dm->Comm.sumReduce( paw );
pan_global = Dm->Comm.sumReduce( pan );
for (int idx=0; idx<3; idx++)
vaw_global(idx) = Dm->Comm.sumReduce( vaw(idx) );
for (int idx=0; idx<3; idx++)
van_global(idx) = Dm->Comm.sumReduce( van(idx));
for (int idx=0; idx<3; idx++)
vawn_global(idx) = Dm->Comm.sumReduce( vawn(idx) );
for (int idx=0; idx<3; idx++)
vawns_global(idx) = Dm->Comm.sumReduce( vawns(idx) );
for (int idx=0; idx<6; idx++){
Gwn_global(idx) = Dm->Comm.sumReduce( Gwn(idx) );
Gns_global(idx) = Dm->Comm.sumReduce( Gns(idx) );
Gws_global(idx) = Dm->Comm.sumReduce( Gws(idx) );
}
trawn_global = Dm->Comm.sumReduce( trawn );
trJwn_global = Dm->Comm.sumReduce( trJwn );
trRwn_global = Dm->Comm.sumReduce( trRwn );
euler_global = Dm->Comm.sumReduce( euler );
An_global = Dm->Comm.sumReduce( An );
Jn_global = Dm->Comm.sumReduce( Jn );
Kn_global = Dm->Comm.sumReduce( Kn );
Dm->Comm.barrier();
// Normalize the phase averages
// (density of both components = 1.0)
@ -1172,6 +1177,8 @@ void TwoPhase::Reduce()
void TwoPhase::NonDimensionalize(double D, double viscosity, double IFT)
{
NULL_USE( viscosity );
NULL_USE( IFT );
awn_global *= D;
ans_global *= D;
ans_global *= D;

2
analysis/TwoPhase.h
View File

@ -12,7 +12,7 @@
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"

83
analysis/analysis.cpp
View File

@ -188,7 +188,7 @@ int ComputeLocalPhaseComponent(const IntArray &PhaseID, int &VALUE, BlobIDArray
/******************************************************************
* Reorder the global blob ids *
******************************************************************/
static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz, MPI_Comm comm )
static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz, const Utilities::MPI& comm )
{
if ( N_blobs==0 )
return 0;
@ -212,7 +212,7 @@ static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int
}
}
ASSERT(max_id<N_blobs);
MPI_Allreduce(local_size,global_size,N_blobs,MPI_DOUBLE,MPI_SUM,comm);
comm.sumReduce(local_size,global_size,N_blobs);
std::vector<std::pair<double,int> > map1(N_blobs);
int N_blobs2 = 0;
for (int i=0; i<N_blobs; i++) {
@ -235,12 +235,12 @@ static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int
PROFILE_STOP("ReorderBlobIDs2",1);
return N_blobs2;
}
void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm )
void ReorderBlobIDs( BlobIDArray& ID, const Utilities::MPI& comm )
{
PROFILE_START("ReorderBlobIDs");
int tmp = ID.max()+1;
int N_blobs = 0;
MPI_Allreduce(&tmp,&N_blobs,1,MPI_INT,MPI_MAX,comm);
N_blobs = comm.maxReduce( tmp );
ReorderBlobIDs2(ID,N_blobs,1,1,1,comm);
PROFILE_STOP("ReorderBlobIDs");
}
@ -260,30 +260,29 @@ static void updateRemoteIds(
int N_send, const std::vector<int>& N_recv,
int64_t *send_buf, std::vector<int64_t*>& recv_buf,
std::map<int64_t,int64_t>& remote_map,
MPI_Comm comm )
const Utilities::MPI& comm )
{
std::vector<MPI_Request> send_req(neighbors.size());
std::vector<MPI_Request> recv_req(neighbors.size());
std::vector<MPI_Status> status(neighbors.size());
std::map<int64_t,global_id_info_struct>::const_iterator it = map.begin();
auto it = map.begin();
ASSERT(N_send==(int)map.size());
for (size_t i=0; i<map.size(); i++, ++it) {
send_buf[2*i+0] = it->first;
send_buf[2*i+1] = it->second.new_id;
}
for (size_t i=0; i<neighbors.size(); i++) {
MPI_Isend( send_buf, 2*N_send, MPI_LONG_LONG, neighbors[i], 0, comm, &send_req[i] );
MPI_Irecv( recv_buf[i], 2*N_recv[i], MPI_LONG_LONG, neighbors[i], 0, comm, &recv_req[i] );
send_req[i] = comm.Isend( send_buf, 2*N_send, neighbors[i], 0 );
recv_req[i] = comm.Irecv( recv_buf[i], 2*N_recv[i], neighbors[i], 0 );
}
for (it=map.begin(); it!=map.end(); ++it) {
remote_map[it->first] = it->second.new_id;
}
for (size_t i=0; i<neighbors.size(); i++) {
MPI_Wait(&recv_req[i],&status[i]);
comm.wait( recv_req[i] );
for (int j=0; j<N_recv[i]; j++)
remote_map[recv_buf[i][2*j+0]] = recv_buf[i][2*j+1];
}
MPI_Waitall(neighbors.size(),getPtr(send_req),getPtr(status));
comm.waitAll(neighbors.size(),getPtr(send_req));
}
// Compute a new local id for each local id
static bool updateLocalIds( const std::map<int64_t,int64_t>& remote_map,
@ -304,18 +303,18 @@ static bool updateLocalIds( const std::map<int64_t,int64_t>& remote_map,
return changed;
}
static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
int nblobs, BlobIDArray& IDs, MPI_Comm comm )
int nblobs, BlobIDArray& IDs, const Utilities::MPI& comm )
{
PROFILE_START("LocalToGlobalIDs",1);
const int rank = rank_info.rank[1][1][1];
int nprocs = comm_size(comm);
int nprocs = comm.getSize();
const int ngx = (IDs.size(0)-nx)/2;
const int ngy = (IDs.size(1)-ny)/2;
const int ngz = (IDs.size(2)-nz)/2;
// Get the number of blobs for each rank
std::vector<int> N_blobs(nprocs,0);
PROFILE_START("LocalToGlobalIDs-Allgather",1);
MPI_Allgather(&nblobs,1,MPI_INT,getPtr(N_blobs),1,MPI_INT,comm);
comm.allGather(nblobs,getPtr(N_blobs));
PROFILE_STOP("LocalToGlobalIDs-Allgather",1);
int64_t N_blobs_tot = 0;
int offset = 0;
@ -363,13 +362,12 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
std::vector<int> N_recv(neighbors.size(),0);
std::vector<MPI_Request> send_req(neighbors.size());
std::vector<MPI_Request> recv_req(neighbors.size());
std::vector<MPI_Status> status(neighbors.size());
for (size_t i=0; i<neighbors.size(); i++) {
MPI_Isend( &N_send, 1, MPI_INT, neighbors[i], 0, comm, &send_req[i] );
MPI_Irecv( &N_recv[i], 1, MPI_INT, neighbors[i], 0, comm, &recv_req[i] );
send_req[i] = comm.Isend( &N_send, 1, neighbors[i], 0 );
recv_req[i] = comm.Irecv( &N_recv[i], 1, neighbors[i], 0 );
}
MPI_Waitall(neighbors.size(),getPtr(send_req),getPtr(status));
MPI_Waitall(neighbors.size(),getPtr(recv_req),getPtr(status));
comm.waitAll(neighbors.size(),getPtr(send_req));
comm.waitAll(neighbors.size(),getPtr(recv_req));
// Allocate memory for communication
int64_t *send_buf = new int64_t[2*N_send];
std::vector<int64_t*> recv_buf(neighbors.size());
@ -398,8 +396,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
bool changed = updateLocalIds( remote_map, map );
// Check if we are finished
int test = changed ? 1:0;
int result = 0;
MPI_Allreduce(&test,&result,1,MPI_INT,MPI_SUM,comm);
int result = comm.sumReduce( test );
if ( result==0 )
break;
}
@ -435,7 +432,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
}
int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS,
BlobIDArray& GlobalBlobID, MPI_Comm comm )
BlobIDArray& GlobalBlobID, const Utilities::MPI& comm )
{
PROFILE_START("ComputeGlobalBlobIDs");
// First compute the local ids
@ -446,7 +443,7 @@ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_inf
return nglobal;
}
int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm )
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, const Utilities::MPI& comm )
{
PROFILE_START("ComputeGlobalPhaseComponent");
// First compute the local ids
@ -462,37 +459,27 @@ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& r
* Compute the mapping of blob ids between timesteps *
******************************************************************/
typedef std::map<BlobIDType,std::map<BlobIDType,int64_t> > map_type;
template<class TYPE> inline MPI_Datatype getMPIType();
template<> inline MPI_Datatype getMPIType<int32_t>() { return MPI_INT; }
template<> inline MPI_Datatype getMPIType<int64_t>() {
if ( sizeof(int64_t)==sizeof(long int) )
return MPI_LONG;
else if ( sizeof(int64_t)==sizeof(double) )
return MPI_DOUBLE;
}
template<class TYPE>
void gatherSet( std::set<TYPE>& set, MPI_Comm comm )
void gatherSet( std::set<TYPE>& set, const Utilities::MPI& comm )
{
int nprocs = comm_size(comm);
MPI_Datatype type = getMPIType<TYPE>();
int nprocs = comm.getSize();
std::vector<TYPE> send_data(set.begin(),set.end());
int send_count = send_data.size();
std::vector<int> recv_count(nprocs,0), recv_disp(nprocs,0);
MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm);
comm.allGather( send_count, getPtr(recv_count) );
for (int i=1; i<nprocs; i++)
recv_disp[i] = recv_disp[i-1] + recv_count[i-1];
std::vector<TYPE> recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]);
MPI_Allgatherv(getPtr(send_data),send_count,type,
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm);
comm.allGather( getPtr(send_data), send_count, getPtr(recv_data),
getPtr(recv_count), getPtr(recv_disp), true );
for (size_t i=0; i<recv_data.size(); i++)
set.insert(recv_data[i]);
}
void gatherSrcIDMap( map_type& src_map, MPI_Comm comm )
void gatherSrcIDMap( map_type& src_map, const Utilities::MPI& comm )
{
int nprocs = comm_size(comm);
MPI_Datatype type = getMPIType<int64_t>();
int nprocs = comm.getSize();
std::vector<int64_t> send_data;
for (map_type::const_iterator it=src_map.begin(); it!=src_map.end(); ++it) {
for (auto it=src_map.begin(); it!=src_map.end(); ++it) {
int id = it->first;
const std::map<BlobIDType,int64_t>& src_ids = it->second;
send_data.push_back(id);
@ -505,21 +492,21 @@ void gatherSrcIDMap( map_type& src_map, MPI_Comm comm )
}
int send_count = send_data.size();
std::vector<int> recv_count(nprocs,0), recv_disp(nprocs,0);
MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm);
comm.allGather(send_count,getPtr(recv_count));
for (int i=1; i<nprocs; i++)
recv_disp[i] = recv_disp[i-1] + recv_count[i-1];
std::vector<int64_t> recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]);
MPI_Allgatherv(getPtr(send_data),send_count,type,
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm);
comm.allGather(getPtr(send_data),send_count,
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),true);
size_t i=0;
src_map.clear();
while ( i < recv_data.size() ) {
BlobIDType id = recv_data[i];
size_t count = recv_data[i+1];
i += 2;
std::map<BlobIDType,int64_t>& src_ids = src_map[id];
auto& src_ids = src_map[id];
for (size_t j=0; j<count; j++,i+=2) {
std::map<BlobIDType,int64_t>::iterator it = src_ids.find(recv_data[i]);
auto it = src_ids.find(recv_data[i]);
if ( it == src_ids.end() )
src_ids.insert(std::pair<BlobIDType,int64_t>(recv_data[i],recv_data[i+1]));
else
@ -538,7 +525,7 @@ void addSrcDstIDs( BlobIDType src_id, map_type& src_map, map_type& dst_map,
}
}
ID_map_struct computeIDMap( int nx, int ny, int nz,
const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm )
const BlobIDArray& ID1, const BlobIDArray& ID2, const Utilities::MPI& comm )
{
ASSERT(ID1.size()==ID2.size());
PROFILE_START("computeIDMap");
@ -780,7 +767,7 @@ void renumberIDs( const std::vector<BlobIDType>& new_ids, BlobIDArray& IDs )
******************************************************************/
void writeIDMap( const ID_map_struct& map, long long int timestep, const std::string& filename )
{
int rank = MPI_WORLD_RANK();
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
if ( rank!=0 )
return;
bool empty = map.created.empty() && map.destroyed.empty() &&

8
analysis/analysis.h
View File

@ -58,7 +58,7 @@ int ComputeLocalPhaseComponent( const IntArray &PhaseID, int &VALUE, IntArray &C
*/
int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS,
BlobIDArray& GlobalBlobID, MPI_Comm comm );
BlobIDArray& GlobalBlobID, const Utilities::MPI& comm );
/*!
@ -75,7 +75,7 @@ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_inf
* @return Return the number of components in the specified phase
*/
int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm );
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, const Utilities::MPI& comm );
/*!
@ -87,7 +87,7 @@ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& r
* @param[in] nz Number of elements in the z-direction
* @param[in/out] ID The ids of the blobs
*/
void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm );
void ReorderBlobIDs( BlobIDArray& ID, const Utilities::MPI& comm );
typedef std::pair<BlobIDType,std::vector<BlobIDType> > BlobIDSplitStruct;
@ -120,7 +120,7 @@ struct ID_map_struct {
* @param[in] ID1 The blob ids at the first timestep
* @param[in] ID2 The blob ids at the second timestep
*/
ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm );
ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, const BlobIDArray& ID2, const Utilities::MPI& comm );
/*!

6
analysis/dcel.cpp
View File

@ -352,6 +352,8 @@ double DECL::EdgeAngle(int edge)
void Isosurface(DoubleArray &A, const double &v)
{
NULL_USE( v );
Point P,Q;
Point PlaceHolder;
Point C0,C1,C2,C3,C4,C5,C6,C7;
@ -562,7 +564,7 @@ void Isosurface(DoubleArray &A, const double &v)
if (P.z == 1.0 && Q.z == 1.0) HalfEdge[idx_edge][3] = -6; // ghost twin for z=1 face
}
// Find all the angles
for (int idx=0; idx<EdgeCount; idx++){
/*for (int idx=0; idx<EdgeCount; idx++){
int V1=HalfEdge[idx][0];
int V2=HalfEdge[idx][1];
int T1= HalfEdge[idx_edge][2];
@ -570,7 +572,7 @@ void Isosurface(DoubleArray &A, const double &v)
if (twin == -1){
}
}
}*/
// Map vertices to global coordinates
for (int idx=0;idx<VertexCount;idx++) {

2
analysis/distance.cpp
View File

@ -176,7 +176,7 @@ void CalcVecDist( Array<Vec> &d, const Array<int> &ID0, const Domain &Dm,
// Update distance
double err = calcVecUpdateInterior( d, dx[0], dx[1], dx[2] );
// Check if we are finished
err = maxReduce( Dm.Comm, err );
err = Dm.Comm.maxReduce( err );
if ( err < tol )
break;
}

186
analysis/morphology.cpp
View File

@ -34,9 +34,6 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
int nx = Dm->Nx;
int ny = Dm->Ny;
int nz = Dm->Nz;
int iproc = Dm->iproc();
int jproc = Dm->jproc();
int kproc = Dm->kproc();
int nprocx = Dm->nprocx();
int nprocy = Dm->nprocy();
int nprocz = Dm->nprocz();
@ -61,11 +58,11 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
}
}
}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
// total Global is the number of nodes in the pore-space
MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm->Comm);
totalGlobal = Dm->Comm.sumReduce( count );
maxdistGlobal = Dm->Comm.sumReduce( maxdist );
double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2);
double volume_fraction=totalGlobal/volume;
if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction);
@ -122,7 +119,6 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
int sendtag,recvtag;
sendtag = recvtag = 7;
int x,y,z;
int ii,jj,kk;
int Nx = nx;
int Ny = ny;
@ -135,9 +131,8 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
// Increase the critical radius until the target saturation is met
double deltaR=0.05; // amount to change the radius in voxel units
double Rcrit_old;
double Rcrit_old=0.0;
double GlobalNumber = 1.f;
int imin,jmin,kmin,imax,jmax,kmax;
if (ErodeLabel == 1){
@ -206,42 +201,24 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag);
//......................................................................................
UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
@ -263,7 +240,7 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id);
//......................................................................................
MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
//double GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
count = 0.f;
for (int k=1; k<Nz-1; k++){
@ -276,7 +253,7 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
}
}
}
MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
countGlobal = Dm->Comm.sumReduce( count );
void_fraction_new = countGlobal/totalGlobal;
void_fraction_diff_new = abs(void_fraction_new-VoidFraction);
/* if (rank==0){
@ -308,7 +285,7 @@ double morph_open()
fillHalo<char> fillChar(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1);
MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
count = 0.f;
for (int k=1; k<Nz-1; k++){
@ -321,7 +298,7 @@ double morph_open()
}
}
}
MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
countGlobal = Dm->Comm.sumReduce( count );
return countGlobal;
}
*/
@ -336,9 +313,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
int nx = Dm->Nx;
int ny = Dm->Ny;
int nz = Dm->Nz;
int iproc = Dm->iproc();
int jproc = Dm->jproc();
int kproc = Dm->kproc();
int nprocx = Dm->nprocx();
int nprocy = Dm->nprocy();
int nprocz = Dm->nprocz();
@ -367,11 +341,11 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
// total Global is the number of nodes in the pore-space
MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm->Comm);
totalGlobal = Dm->Comm.sumReduce( count );
maxdistGlobal = Dm->Comm.sumReduce( maxdist );
double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2);
double volume_fraction=totalGlobal/volume;
if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction);
@ -427,7 +401,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
int sendtag,recvtag;
sendtag = recvtag = 7;
int x,y,z;
int ii,jj,kk;
int Nx = nx;
int Ny = ny;
@ -442,7 +415,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
double deltaR=0.05; // amount to change the radius in voxel units
double Rcrit_old;
double GlobalNumber = 1.f;
int imin,jmin,kmin,imax,jmax,kmax;
double Rcrit_new = maxdistGlobal;
@ -450,7 +422,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
// Rcrit_new = strtod(argv[2],NULL);
// if (rank==0) printf("Max. distance =%f, Initial critical radius = %f \n",maxdistGlobal,Rcrit_new);
//}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
FILE *DRAIN = fopen("morphdrain.csv","w");
@ -516,42 +488,42 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag);
//......................................................................................
UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
@ -572,7 +544,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id);
UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id);
//......................................................................................
MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
// double GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
@ -591,7 +563,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
BlobIDstruct new_index;
double vF=0.0; double vS=0.0;
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
@ -627,7 +599,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){
@ -653,7 +625,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
}
}
MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
countGlobal = Dm->Comm.sumReduce( count );
void_fraction_new = countGlobal/totalGlobal;
void_fraction_diff_new = abs(void_fraction_new-VoidFraction);
if (rank==0){
@ -693,17 +665,11 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
return final_void_fraction;
}
double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetGrowth){
double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetGrowth)
{
int Nx = Dm->Nx;
int Ny = Dm->Ny;
int Nz = Dm->Nz;
int iproc = Dm->iproc();
int jproc = Dm->jproc();
int kproc = Dm->kproc();
int nprocx = Dm->nprocx();
int nprocy = Dm->nprocy();
int nprocz = Dm->nprocz();
int rank = Dm->rank();
double count=0.0;
@ -716,7 +682,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
}
}
}
double count_original=sumReduce( Dm->Comm, count);
double count_original = Dm->Comm.sumReduce( count);
// Estimate morph_delta
double morph_delta = 0.0;
@ -746,8 +712,8 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
}
}
}
count=sumReduce( Dm->Comm, count);
MAX_DISPLACEMENT = maxReduce( Dm->Comm, MAX_DISPLACEMENT);
count = Dm->Comm.sumReduce( count );
MAX_DISPLACEMENT = Dm->Comm.maxReduce( MAX_DISPLACEMENT );
GrowthEstimate = count - count_original;
ERROR = fabs((GrowthEstimate-TargetGrowth) /TargetGrowth);
@ -790,7 +756,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
}
}
}
count=sumReduce( Dm->Comm, count);
count = Dm->Comm.sumReduce( count );
return count;
}

60
analysis/runAnalysis.cpp
View File

@ -3,7 +3,7 @@
#include "analysis/analysis.h"
#include "common/Array.h"
#include "common/Communication.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/ScaLBL.h"
#include "models/ColorModel.h"
@ -462,7 +462,7 @@ private:
/******************************************************************
* MPI comm wrapper for use with analysis *
******************************************************************/
runAnalysis::commWrapper::commWrapper( int tag_, MPI_Comm comm_, runAnalysis* analysis_ ):
runAnalysis::commWrapper::commWrapper( int tag_, const Utilities::MPI& comm_, runAnalysis* analysis_ ):
comm(comm_),
tag(tag_),
analysis(analysis_)
@ -479,7 +479,7 @@ runAnalysis::commWrapper::~commWrapper()
{
if ( tag == -1 )
return;
MPI_Barrier( comm );
comm.barrier();
analysis->d_comm_used[tag] = false;
}
runAnalysis::commWrapper runAnalysis::getComm( )
@ -496,10 +496,10 @@ runAnalysis::commWrapper runAnalysis::getComm( )
if ( tag == -1 )
ERROR("Unable to get comm");
}
MPI_Bcast( &tag, 1, MPI_INT, 0, d_comm );
tag = d_comm.bcast( tag, 0 );
d_comm_used[tag] = true;
if ( d_comms[tag] == MPI_COMM_NULL )
MPI_Comm_dup( MPI_COMM_WORLD, &d_comms[tag] );
if ( d_comms[tag].isNull() )
d_comms[tag] = d_comm.dup();
return commWrapper(tag,d_comms[tag],this);
}
@ -507,14 +507,20 @@ runAnalysis::commWrapper runAnalysis::getComm( )
/******************************************************************
* Constructor/Destructors *
******************************************************************/
runAnalysis::runAnalysis(std::shared_ptr<Database> input_db, const RankInfoStruct& rank_info, std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> Dm,
int Np, bool Regular, IntArray Map ):
d_Np( Np ),
d_regular ( Regular),
d_rank_info( rank_info ),
d_Map( Map ),
d_fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1),
d_ScaLBL_Comm( ScaLBL_Comm)
runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
const RankInfoStruct& rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm,
std::shared_ptr <Domain> Dm,
int Np,
bool Regular,
IntArray Map ):
d_Np( Np ),
d_regular ( Regular),
d_rank_info( rank_info ),
d_Map( Map ),
d_fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1),
d_comm( Utilities::MPI( MPI_COMM_WORLD ).dup() ),
d_ScaLBL_Comm( ScaLBL_Comm)
{
auto db = input_db->getDatabase( "Analysis" );
@ -552,7 +558,7 @@ runAnalysis::runAnalysis(std::shared_ptr<Database> input_db, const RankInfoStruc
d_restartFile = restart_file + "." + rankString;
d_rank = MPI_WORLD_RANK();
d_rank = d_comm.getRank();
writeIDMap(ID_map_struct(),0,id_map_filename);
// Initialize IO for silo
IO::initialize("","silo","false");
@ -621,11 +627,8 @@ runAnalysis::runAnalysis(std::shared_ptr<Database> input_db, const RankInfoStruc
// Initialize the comms
MPI_Comm_dup(MPI_COMM_WORLD,&d_comm);
for (int i=0; i<1024; i++) {
d_comms[i] = MPI_COMM_NULL;
for (int i=0; i<1024; i++)
d_comm_used[i] = false;
}
// Initialize the threads
int N_threads = db->getWithDefault<int>( "N_threads", 4 );
auto method = db->getWithDefault<std::string>( "load_balance", "default" );
@ -635,12 +638,6 @@ runAnalysis::~runAnalysis( )
{
// Finish processing analysis
finish();
// Clear internal data
MPI_Comm_free( &d_comm );
for (int i=0; i<1024; i++) {
if ( d_comms[i] != MPI_COMM_NULL )
MPI_Comm_free(&d_comms[i]);
}
}
void runAnalysis::finish( )
{
@ -654,7 +651,7 @@ void runAnalysis::finish( )
d_wait_subphase.reset();
d_wait_restart.reset();
// Syncronize
MPI_Barrier( d_comm );
d_comm.barrier();
PROFILE_STOP("finish");
}
@ -767,6 +764,8 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
double *Pressure, double *Velocity, double *fq, double *Den)
{
int N = d_N[0]*d_N[1]*d_N[2];
NULL_USE( N );
NULL_USE( Phi );
auto db = input_db->getDatabase( "Analysis" );
//int timestep = db->getWithDefault<int>( "timestep", 0 );
@ -937,8 +936,6 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
******************************************************************/
void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den)
{
int N = d_N[0]*d_N[1]*d_N[2];
// Check which analysis steps we need to perform
auto color_db = input_db->getDatabase( "Color" );
auto vis_db = input_db->getDatabase( "Visualization" );
@ -954,7 +951,7 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
finish();
}
PROFILE_START("run");
PROFILE_START("basic");
// Copy the appropriate variables to the host (so we can spawn new threads)
ScaLBL_DeviceBarrier();
@ -983,7 +980,6 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
}
PROFILE_STOP("Copy data to host");
PROFILE_START("run",1);
// Spawn threads to do the analysis work
//if (timestep%d_restart_interval==0){
// if ( matches(type,AnalysisType::ComputeAverages) ) {
@ -1036,12 +1032,11 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
d_wait_vis = d_tpool.add_work(work);
}
PROFILE_STOP("run");
PROFILE_STOP("basic");
}
void runAnalysis::WriteVisData(int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den)
{
int N = d_N[0]*d_N[1]*d_N[2];
auto color_db = input_db->getDatabase( "Color" );
auto vis_db = input_db->getDatabase( "Visualization" );
//int timestep = color_db->getWithDefault<int>( "timestep", 0 );
@ -1068,7 +1063,6 @@ void runAnalysis::WriteVisData(int timestep, std::shared_ptr<Database> input_db,
d_wait_vis = d_tpool.add_work(work2);
//Averages.WriteVis = false;
// }
PROFILE_STOP("write vis");
}

8
analysis/runAnalysis.h
View File

@ -68,10 +68,10 @@ public:
class commWrapper
{
public:
MPI_Comm comm;
Utilities::MPI comm;
int tag;
runAnalysis *analysis;
commWrapper( int tag, MPI_Comm comm, runAnalysis *analysis );
commWrapper( int tag, const Utilities::MPI& comm, runAnalysis *analysis );
commWrapper( ) = delete;
commWrapper( const commWrapper &rhs ) = delete;
commWrapper& operator=( const commWrapper &rhs ) = delete;
@ -100,8 +100,8 @@ private:
std::vector<IO::MeshDataStruct> d_meshData;
fillHalo<double> d_fillData;
std::string d_restartFile;
MPI_Comm d_comm;
MPI_Comm d_comms[1024];
Utilities::MPI d_comm;
Utilities::MPI d_comms[1024];
volatile bool d_comm_used[1024];
std::shared_ptr<ScaLBL_Communicator> d_ScaLBL_Comm;

13
analysis/uCT.cpp
View File

@ -157,6 +157,7 @@ void solve( const Array<float>& VOL, Array<float>& Mean, Array<char>& ID,
// int depth = 5;
// float sigsq=0.1;
int nlm_count = NLM3D( MultiScaleSmooth, Mean, Dist, NonLocalMean, depth, sigsq);
NULL_USE( nlm_count );
fillFloat.fill(NonLocalMean);
}
@ -201,6 +202,7 @@ void refine( const Array<float>& Dist_coarse,
// int depth = 3;
// float sigsq = 0.1;
int nlm_count = NLM3D( MultiScaleSmooth, Mean, Dist, NonLocalMean, depth, sigsq);
NULL_USE( nlm_count );
fillFloat.fill(NonLocalMean);
segment( NonLocalMean, ID, 0.001 );
for (size_t i=0; i<ID.length(); i++) {
@ -226,8 +228,7 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
Array<float>& Mean, Array<float>& Dist1, Array<float>& Dist2 )
{
PROFILE_SCOPED(timer,"filter_final");
int rank;
MPI_Comm_rank(Dm.Comm,&rank);
int rank = Dm.Comm.getRank();
int Nx = Dm.Nx-2;
int Ny = Dm.Ny-2;
int Nz = Dm.Nz-2;
@ -240,7 +241,7 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
float tmp = 0;
for (size_t i=0; i<Dist0.length(); i++)
tmp += Dist0(i)*Dist0(i);
tmp = sqrt( sumReduce(Dm.Comm,tmp) / sumReduce(Dm.Comm,(float)Dist0.length()) );
tmp = sqrt( Dm.Comm.sumReduce(tmp) / Dm.Comm.sumReduce<float>(Dist0.length()) );
const float dx1 = 0.3*tmp;
const float dx2 = 1.05*dx1;
if (rank==0)
@ -283,7 +284,7 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
Phase.fill(1);
ComputeGlobalBlobIDs( Nx, Ny, Nz, Dm.rank_info, Phase, SignDist, 0, 0, GlobalBlobID, Dm.Comm );
fillInt.fill(GlobalBlobID);
int N_blobs = maxReduce(Dm.Comm,GlobalBlobID.max()+1);
int N_blobs = Dm.Comm.maxReduce(GlobalBlobID.max()+1);
std::vector<float> mean(N_blobs,0);
std::vector<int> count(N_blobs,0);
for (int k=1; k<=Nz; k++) {
@ -319,8 +320,8 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
}
}
}
mean = sumReduce(Dm.Comm,mean);
count = sumReduce(Dm.Comm,count);
mean = Dm.Comm.sumReduce(mean);
count = Dm.Comm.sumReduce(count);
for (size_t i=0; i<mean.size(); i++)
mean[i] /= count[i];
/*if (rank==0) {

579
cmake/FindHIP.cmake Normal file
View File

@ -0,0 +1,579 @@
###############################################################################
# FindHIP.cmake
###############################################################################
###############################################################################
# SET: Variable defaults
###############################################################################
# User defined flags
set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_NVCC_FLAGS)
set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
list(REMOVE_DUPLICATES _hip_configuration_types)
foreach(config ${_hip_configuration_types})
string(TOUPPER ${config} config_upper)
set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
endforeach()
option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file. With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
mark_as_advanced(HIP_HOST_COMPILATION_CPP)
###############################################################################
# Set HIP CMAKE Flags
###############################################################################
# Copy the invocation styles from CXX to HIP
set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
# Set the CMake Flags to use the HCC Compilier.
set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_PATH} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_PATH} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_PATH} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
###############################################################################
# FIND: HIP and associated helper binaries
###############################################################################
# HIP is supported on Linux only
if(UNIX AND NOT APPLE AND NOT CYGWIN)
# Search for HIP installation
if(NOT HIP_ROOT_DIR)
# Search in user specified path first
find_path(
HIP_ROOT_DIR
NAMES hipconfig
PATHS
ENV ROCM_PATH
ENV HIP_PATH
PATH_SUFFIXES bin
DOC "HIP installed location"
NO_DEFAULT_PATH
)
# Now search in default path
find_path(
HIP_ROOT_DIR
NAMES hipconfig
PATHS
/opt/rocm
/opt/rocm/hip
PATH_SUFFIXES bin
DOC "HIP installed location"
)
# Check if we found HIP installation
if(HIP_ROOT_DIR)
# If so, fix the path
string(REGEX REPLACE "[/\\\\]?bin[64]*[/\\\\]?$" "" HIP_ROOT_DIR ${HIP_ROOT_DIR})
# And push it back to the cache
set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
endif()
if(NOT EXISTS ${HIP_ROOT_DIR})
if(HIP_FIND_REQUIRED)
message(FATAL_ERROR "Specify HIP_ROOT_DIR")
elseif(NOT HIP_FIND_QUIETLY)
message("HIP_ROOT_DIR not found or specified")
endif()
endif()
endif()
# Find HIPCC executable
find_program(
HIP_HIPCC_EXECUTABLE
NAMES hipcc
PATHS
"${HIP_ROOT_DIR}"
ENV ROCM_PATH
ENV HIP_PATH
/opt/rocm
/opt/rocm/hip
PATH_SUFFIXES bin
NO_DEFAULT_PATH
)
if(NOT HIP_HIPCC_EXECUTABLE)
# Now search in default paths
find_program(HIP_HIPCC_EXECUTABLE hipcc)
endif()
mark_as_advanced(HIP_HIPCC_EXECUTABLE)
# Find HIPCONFIG executable
find_program(
HIP_HIPCONFIG_EXECUTABLE
NAMES hipconfig
PATHS
"${HIP_ROOT_DIR}"
ENV ROCM_PATH
ENV HIP_PATH
/opt/rocm
/opt/rocm/hip
PATH_SUFFIXES bin
NO_DEFAULT_PATH
)
if(NOT HIP_HIPCONFIG_EXECUTABLE)
# Now search in default paths
find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
endif()
mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
# Find HIPCC_CMAKE_LINKER_HELPER executable
find_program(
HIP_HIPCC_CMAKE_LINKER_HELPER
NAMES hipcc_cmake_linker_helper
PATHS
"${HIP_ROOT_DIR}"
ENV ROCM_PATH
ENV HIP_PATH
/opt/rocm
/opt/rocm/hip
PATH_SUFFIXES bin
NO_DEFAULT_PATH
)
if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
# Now search in default paths
find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
endif()
mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
# Compute the version
execute_process(
COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
OUTPUT_VARIABLE _hip_version
ERROR_VARIABLE _hip_error
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_STRIP_TRAILING_WHITESPACE
)
if(NOT _hip_error)
set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
else()
set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
endif()
mark_as_advanced(HIP_VERSION)
endif()
if(HIP_VERSION)
string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
list(GET _hip_version_list 1 HIP_VERSION_MINOR)
list(GET _hip_version_list 2 HIP_VERSION_PATCH)
set(HIP_VERSION_STRING "${HIP_VERSION}")
endif()
if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
# Compute the platform
execute_process(
COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
OUTPUT_VARIABLE _hip_platform
OUTPUT_STRIP_TRAILING_WHITESPACE
)
set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
mark_as_advanced(HIP_PLATFORM)
endif()
endif()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(
HIP
REQUIRED_VARS
HIP_ROOT_DIR
HIP_HIPCC_EXECUTABLE
HIP_HIPCONFIG_EXECUTABLE
HIP_PLATFORM
VERSION_VAR HIP_VERSION
)
###############################################################################
# MACRO: Locate helper files
###############################################################################
macro(HIP_FIND_HELPER_FILE _name _extension)
set(_hip_full_name "${_name}.${_extension}")
get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
if(NOT EXISTS "${HIP_${_name}}")
set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
if(HIP_FIND_REQUIRED)
message(FATAL_ERROR "${error_message}")
else()
if(NOT HIP_FIND_QUIETLY)
message(STATUS "${error_message}")
endif()
endif()
endif()
# Set this variable as internal, so the user isn't bugged with it.
set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
endmacro()
###############################################################################
hip_find_helper_file(run_make2cmake cmake)
hip_find_helper_file(run_hipcc cmake)
###############################################################################
###############################################################################
# MACRO: Reset compiler flags
###############################################################################
macro(HIP_RESET_FLAGS)
unset(HIP_HIPCC_FLAGS)
unset(HIP_HCC_FLAGS)
unset(HIP_NVCC_FLAGS)
foreach(config ${_hip_configuration_types})
string(TOUPPER ${config} config_upper)
unset(HIP_HIPCC_FLAGS_${config_upper})
unset(HIP_HCC_FLAGS_${config_upper})
unset(HIP_NVCC_FLAGS_${config_upper})
endforeach()
endmacro()
###############################################################################
# MACRO: Separate the options from the sources
###############################################################################
macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _nvcc_options)
set(${_sources})
set(${_cmake_options})
set(${_hipcc_options})
set(${_hcc_options})
set(${_nvcc_options})
set(_hipcc_found_options FALSE)
set(_hcc_found_options FALSE)
set(_nvcc_found_options FALSE)
foreach(arg ${ARGN})
if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
set(_hipcc_found_options TRUE)
set(_hcc_found_options FALSE)
set(_nvcc_found_options FALSE)
elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
set(_hipcc_found_options FALSE)
set(_hcc_found_options TRUE)
set(_nvcc_found_options FALSE)
elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
set(_hipcc_found_options FALSE)
set(_hcc_found_options FALSE)
set(_nvcc_found_options TRUE)
elseif(
"x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
"x${arg}" STREQUAL "xSTATIC" OR
"x${arg}" STREQUAL "xSHARED" OR
"x${arg}" STREQUAL "xMODULE"
)
list(APPEND ${_cmake_options} ${arg})
else()
if(_hipcc_found_options)
list(APPEND ${_hipcc_options} ${arg})
elseif(_hcc_found_options)
list(APPEND ${_hcc_options} ${arg})
elseif(_nvcc_found_options)
list(APPEND ${_nvcc_options} ${arg})
else()
# Assume this is a file
list(APPEND ${_sources} ${arg})
endif()
endif()
endforeach()
endmacro()
###############################################################################
# MACRO: Add include directories to pass to the hipcc command
###############################################################################
set(HIP_HIPCC_INCLUDE_ARGS_USER "")
macro(HIP_INCLUDE_DIRECTORIES)
foreach(dir ${ARGN})
list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
endforeach()
endmacro()
###############################################################################
# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
###############################################################################
function(HIP_COMPUTE_BUILD_PATH path build_path)
# Convert to cmake style paths
file(TO_CMAKE_PATH "${path}" bpath)
if(IS_ABSOLUTE "${bpath}")
string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
if(_binary_dir_pos EQUAL 0)
file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
else()
file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
endif()
endif()
# Remove leading /
string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
# Avoid absolute paths by removing ':'
string(REPLACE ":" "_" bpath "${bpath}")
# Avoid relative paths that go up the tree
string(REPLACE "../" "__/" bpath "${bpath}")
# Avoid spaces
string(REPLACE " " "_" bpath "${bpath}")
# Strip off the filename
get_filename_component(bpath "${bpath}" PATH)
set(${build_path} "${bpath}" PARENT_SCOPE)
endfunction()
###############################################################################
# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
###############################################################################
macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
set(_hip_found_config)
foreach(arg ${ARGN})
# Determine if we are dealing with a per-configuration flag
foreach(config ${_hip_configuration_types})
string(TOUPPER ${config} config_upper)
if(arg STREQUAL "${config_upper}")
set(_hip_found_config _${arg})
# Clear arg to prevent it from being processed anymore
set(arg)
endif()
endforeach()
if(arg)
list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
endif()
endforeach()
endmacro()
###############################################################################
# MACRO: Try and include dependency file if it exists
###############################################################################
macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
set(HIP_HIPCC_DEPEND)
set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
# Create the dependency file if it doesn't exist
if(NOT EXISTS ${dependency_file})
file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
endif()
# Include the dependency file
include(${dependency_file})
# Verify the existence of all the included files
if(HIP_HIPCC_DEPEND)
foreach(f ${HIP_HIPCC_DEPEND})
if(NOT EXISTS ${f})
# If they aren't there, regenerate the file again
set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
endif()
endforeach()
else()
# No dependencies, so regenerate the file
set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
endif()
# Regenerate the dependency file if needed
if(HIP_HIPCC_DEPEND_REGENERATE)
set(HIP_HIPCC_DEPEND ${dependency_file})
file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
endif()
endmacro()
###############################################################################
# MACRO: Prepare cmake commands for the target
###############################################################################
macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
set(_hip_flags "")
string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
if(HIP_HOST_COMPILATION_CPP)
set(HIP_C_OR_CXX CXX)
else()
set(HIP_C_OR_CXX C)
endif()
set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
# Initialize list of includes with those specified by the user. Append with
# ones specified to cmake directly.
set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
# Add the include directories
set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
list(REMOVE_DUPLICATES _hip_include_directories)
if(_hip_include_directories)
foreach(dir ${_hip_include_directories})
list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
endforeach()
endif()
HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN})
HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
# Add the compile definitions
set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
# Check if we are building shared library.
set(_hip_build_shared_libs FALSE)
list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
set(_hip_build_shared_libs TRUE)
endif()
list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
if(_hip_found_STATIC GREATER -1)
set(_hip_build_shared_libs FALSE)
endif()
# If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
if(_hip_build_shared_libs)
list(APPEND HIP_HCC_FLAGS "-fPIC")
list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
endif()
# Set host compiler
set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
# Set compiler flags
set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
foreach(config ${_hip_configuration_types})
string(TOUPPER ${config} config_upper)
set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
endforeach()
# Reset the output variable
set(_hip_generated_files "")
set(_hip_source_files "")
# Iterate over all arguments and create custom commands for all source files
foreach(file ${ARGN})
# Ignore any file marked as a HEADER_FILE_ONLY
get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
# Allow per source file overrides of the format. Also allows compiling non .cu files.
get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
set(host_flag FALSE)
else()
set(host_flag TRUE)
endif()
if(NOT host_flag)
# Determine output directory
HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
get_filename_component(basename ${file} NAME)
set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
# Set file names
set(generated_file "${generated_file_path}/${generated_file_basename}")
set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
# Set properties for object files
set_source_files_properties("${generated_file}"
PROPERTIES
EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
)
# Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
get_filename_component(file_path "${file}" PATH)
if(IS_ABSOLUTE "${file_path}")
set(source_file "${file}")
else()
set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
endif()
# Bring in the dependencies
HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
# Configure the build script
configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
file(GENERATE
OUTPUT "${custom_target_script}"
INPUT "${custom_target_script_pregen}"
)
set(main_dep DEPENDS ${source_file})
if(CMAKE_GENERATOR MATCHES "Makefiles")
set(verbose_output "$(VERBOSE)")
elseif(HIP_VERBOSE_BUILD)
set(verbose_output ON)
else()
set(verbose_output OFF)
endif()
# Create up the comment string
file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
# Build the generated file and dependency file
add_custom_command(
OUTPUT ${generated_file}
# These output files depend on the source_file and the contents of cmake_dependency_file
${main_dep}
DEPENDS ${HIP_HIPCC_DEPEND}
DEPENDS ${custom_target_script}
# Make sure the output directory exists before trying to write to it.
COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
COMMAND ${CMAKE_COMMAND} ARGS
-D verbose:BOOL=${verbose_output}
-D build_configuration:STRING=${_hip_build_configuration}
-D "generated_file:STRING=${generated_file}"
-P "${custom_target_script}"
WORKING_DIRECTORY "${hip_compile_output_dir}"
COMMENT "${hip_build_comment_string}"
)
# Make sure the build system knows the file is generated
set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
list(APPEND _hip_generated_files ${generated_file})
list(APPEND _hip_source_files ${file})
endif()
endforeach()
# Set the return parameter
set(${_generated_files} ${_hip_generated_files})
set(${_source_files} ${_hip_source_files})
endmacro()
###############################################################################
# HIP_ADD_EXECUTABLE
###############################################################################
macro(HIP_ADD_EXECUTABLE hip_target)
# Separate the sources from the options
HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN})
HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
if(_source_files)
list(REMOVE_ITEM _sources ${_source_files})
endif()
if("x${HCC_HOME}" STREQUAL "x")
set(HCC_HOME "/opt/rocm/hcc")
endif()
set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
endmacro()
###############################################################################
# HIP_ADD_LIBRARY
###############################################################################
macro(HIP_ADD_LIBRARY hip_target)
# Separate the sources from the options
HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN})
HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
if(_source_files)
list(REMOVE_ITEM _sources ${_source_files})
endif()
add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
endmacro()
# vim: ts=4:sw=4:expandtab:smartindent

6
cmake/libraries.cmake
View File

@ -77,7 +77,7 @@ MACRO( CONFIGURE_MPI )
ENDIF ()
ELSE ()
# Search for the MPI executable in the current directory
FIND_PROGRAM ( MPIEXEC NAMES mpiexec mpirun lamexec PATHS ${MPI_DIRECTORY}/bin NO_DEFAULT_PATH )
FIND_PROGRAM( MPIEXEC NAMES mpiexec mpirun lamexec PATHS ${MPI_DIRECTORY}/bin NO_DEFAULT_PATH )
IF ( NOT MPIEXEC )
MESSAGE( FATAL_ERROR "Could not locate mpi executable" )
ENDIF()
@ -305,4 +305,8 @@ MACRO ( CONFIGURE_LBPM )
SET( CMAKE_BUILD_WITH_INSTALL_RPATH TRUE )
SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}" "${LBPM_INSTALL_DIR}/lib" )
ENDIF()
# Suppress some common warnings
IF ( USING_GCC )
SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder -Wno-unused-parameter")
ENDIF()
ENDMACRO ()

10
cmake/macros.cmake
View File

@ -848,7 +848,7 @@ FUNCTION( ADD_${PROJ}_TEST EXEFILE ${ARGN} )
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
CREATE_TEST_NAME( ${EXEFILE} ${ARGN} )
IF ( USE_MPI_FOR_SERIAL_TESTS )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
@ -877,7 +877,7 @@ FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
ELSEIF( ${PROCS} STREQUAL "1" )
CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
IF ( USE_MPI_FOR_SERIAL_TESTS )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
@ -909,7 +909,7 @@ FUNCTION( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
ELSEIF ( ${PROCS} GREATER ${TEST_MAX_PROCS} )
MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
@ -930,7 +930,7 @@ MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
ELSEIF ( USE_MPI OR USE_EXT_MPI )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
@ -966,7 +966,7 @@ FUNCTION( ADD_${PROJ}_EXAMPLE EXEFILE PROCS ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ELSEIF ( USE_EXT_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
CREATE_TEST_NAME( "example--${EXEFILE}_${PROCS}procs" ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ENDIF()
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )

226
common/Communication.h
View File

@ -1,7 +1,7 @@
#ifndef COMMUNICATION_H_INC
#define COMMUNICATION_H_INC
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "common/Array.h"
@ -38,7 +38,7 @@ struct RankInfoStruct {
//! Redistribute domain data (dst may be smaller than the src)
template<class TYPE>
Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm );
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, const Utilities::MPI& comm );
/*!
@ -59,7 +59,7 @@ public:
* @param[in] fill Fill {faces,edges,corners}
* @param[in] periodic Periodic dimensions
*/
fillHalo( MPI_Comm comm, const RankInfoStruct& info,
fillHalo( const Utilities::MPI& comm, const RankInfoStruct& info,
std::array<int,3> n, std::array<int,3> ng, int tag, int depth,
std::array<bool,3> fill = {true,true,true},
std::array<bool,3> periodic = {true,true,true} );
@ -83,7 +83,7 @@ public:
private:
MPI_Comm comm;
Utilities::MPI comm;
RankInfoStruct info;
std::array<int,3> n, ng;
int depth;
@ -93,8 +93,6 @@ private:
TYPE *mem;
TYPE *send[3][3][3], *recv[3][3][3];
MPI_Request send_req[3][3][3], recv_req[3][3][3];
size_t N_type;
MPI_Datatype datatype;
fillHalo(); // Private empty constructor
fillHalo(const fillHalo&); // Private copy constructor
fillHalo& operator=(const fillHalo&); // Private assignment operator
@ -136,7 +134,7 @@ void InitializeRanks( const int rank, const int nprocx, const int nprocy, const
//***************************************************************************************
inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int recvtag,
inline void CommunicateSendRecvCounts( const Utilities::MPI& comm, int sendtag, int recvtag,
int rank_x, int rank_y, int rank_z,
int rank_X, int rank_Y, int rank_Z,
int rank_xy, int rank_XY, int rank_xY, int rank_Xy,
@ -154,54 +152,53 @@ inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int r
int& recvCount_yz, int& recvCount_YZ, int& recvCount_yZ, int& recvCount_Yz )
{
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]);
req1[0] = comm.Isend(&sendCount_x,1,rank_x,sendtag+0);
req2[0] = comm.Irecv(&recvCount_X,1,rank_X,recvtag+0);
req1[1] = comm.Isend(&sendCount_X,1,rank_X,sendtag+1);
req2[1] = comm.Irecv(&recvCount_x,1,rank_x,recvtag+1);
req1[2] = comm.Isend(&sendCount_y,1,rank_y,sendtag+2);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_Y,recvtag+2);
req1[3] = comm.Isend(&sendCount_Y,1,rank_Y,sendtag+3);
req2[3] = comm.Irecv(&recvCount_y,1,rank_y,recvtag+3);
req1[4] = comm.Isend(&sendCount_z,1,rank_z,sendtag+4);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_Z,recvtag+4);
req1[5] = comm.Isend(&sendCount_Z,1,rank_Z,sendtag+5);
req2[5] = comm.Irecv(&recvCount_z,1,rank_z,recvtag+5);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_xy,sendtag+6);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_XY,recvtag+6);
req1[7] = comm.Isend(&sendCount_XY,1,rank_XY,sendtag+7);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_xy,recvtag+7);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_Xy,sendtag+8);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_xY,recvtag+8);
req1[9] = comm.Isend(&sendCount_xY,1,rank_xY,sendtag+9);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_Xy,recvtag+9);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_xz,sendtag+10);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_XZ,recvtag+10);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_XZ,sendtag+11);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_xz,recvtag+11);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_Xz,sendtag+12);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_xZ,recvtag+12);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_xZ,sendtag+13);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_Xz,recvtag+13);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(Communicator);
req1[14] = comm.Isend(&sendCount_yz,1,rank_yz,sendtag+14);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_YZ,recvtag+14);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_YZ,sendtag+15);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_yz,recvtag+15);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_Yz,sendtag+16);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_yZ,recvtag+16);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_yZ,sendtag+17);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_Yz,recvtag+17);
comm.waitAll( 18, req1 );
comm.waitAll( 18, req2 );
comm.barrier();
}
//***************************************************************************************
inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvtag,
inline void CommunicateRecvLists( const Utilities::MPI& comm, int sendtag, int recvtag,
int *sendList_x, int *sendList_y, int *sendList_z, int *sendList_X, int *sendList_Y, int *sendList_Z,
int *sendList_xy, int *sendList_XY, int *sendList_xY, int *sendList_Xy,
int *sendList_xz, int *sendList_XZ, int *sendList_xZ, int *sendList_Xz,
@ -222,53 +219,52 @@ inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvta
int rank_Xy, int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz, int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz)
{
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz,recvtag);
comm.waitAll( 18, req1 );
comm.waitAll( 18, req2 );
}
//***************************************************************************************
inline void CommunicateMeshHalo(DoubleArray &Mesh, MPI_Comm Communicator,
inline void CommunicateMeshHalo(DoubleArray &Mesh, const Utilities::MPI& comm,
double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy,
double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz,
@ -318,42 +314,24 @@ inline void CommunicateMeshHalo(DoubleArray &Mesh, MPI_Comm Communicator,
PackMeshData(sendList_yZ, sendCount_yZ ,sendbuf_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData);
//......................................................................................
MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_DOUBLE,rank_x,sendtag,
recvbuf_X,recvCount_X,MPI_DOUBLE,rank_X,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_DOUBLE,rank_X,sendtag,
recvbuf_x,recvCount_x,MPI_DOUBLE,rank_x,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_DOUBLE,rank_y,sendtag,
recvbuf_Y,recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,
recvbuf_y,recvCount_y,MPI_DOUBLE,rank_y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_DOUBLE,rank_z,sendtag,
recvbuf_Z,recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,
recvbuf_z,recvCount_z,MPI_DOUBLE,rank_z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,
recvbuf_XY,recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,
recvbuf_xy,recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,
recvbuf_xY,recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,
recvbuf_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,
recvbuf_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,
recvbuf_xz,recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,
recvbuf_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,
recvbuf_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,
recvbuf_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,
recvbuf_yz,recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,
recvbuf_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,
recvbuf_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,Communicator,MPI_STATUS_IGNORE);
comm.sendrecv(sendbuf_x,sendCount_x,rank_x,sendtag,recvbuf_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendbuf_X,sendCount_X,rank_X,sendtag,recvbuf_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendbuf_y,sendCount_y,rank_y,sendtag,recvbuf_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendbuf_Y,sendCount_Y,rank_Y,sendtag,recvbuf_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendbuf_z,sendCount_z,rank_z,sendtag,recvbuf_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendbuf_Z,sendCount_Z,rank_Z,sendtag,recvbuf_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendbuf_xy,sendCount_xy,rank_xy,sendtag,recvbuf_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendbuf_XY,sendCount_XY,rank_XY,sendtag,recvbuf_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag,recvbuf_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendbuf_xY,sendCount_xY,rank_xY,sendtag,recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendbuf_xz,sendCount_xz,rank_xz,sendtag,recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag,recvbuf_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag,recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag,recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendbuf_yz,sendCount_yz,rank_yz,sendtag,recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag,recvbuf_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag,recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag,recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData);

65
common/Communication.hpp
View File

@ -2,9 +2,8 @@
#define COMMUNICATION_HPP_INC
#include "common/Communication.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
//#include "ProfilerApp.h"
/********************************************************
@ -12,17 +11,19 @@
********************************************************/
template<class TYPE>
Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm )
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, const Utilities::MPI& comm )
{
#ifdef USE_MPI
if ( comm.getSize() == 1 ) {
return src_data.subset( { 0, (size_t) dst_size[0]-1, 0, (size_t) dst_size[1]-1, 0, (size_t) dst_size[2]-1 } );
}
// Get the src size
std::array<int,3> src_size;
int size0[3] = { (int) src_data.size(0), (int) src_data.size(1), (int) src_data.size(2) };
MPI_Allreduce( size0, src_size.data(), 3, MPI_INT, MPI_MAX, comm );
comm.maxReduce( size0, src_size.data(), 3 );
if ( !src_data.empty() )
ASSERT( src_size[0] == size0[0] && src_size[1] == size0[1] && src_size[2] == size0[2] );
// Check that dst_size matches on all ranks
MPI_Allreduce( dst_size.data(), size0, 3, MPI_INT, MPI_MAX, comm );
comm.maxReduce( dst_size.data(), size0, 3 );
ASSERT( dst_size[0] == size0[0] && dst_size[1] == size0[1] && dst_size[2] == size0[2] );
// Function to get overlap range
auto calcOverlap = []( int i1[3], int i2[3], int j1[3], int j2[3] ) {
@ -44,9 +45,9 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
if ( !src_data.empty() ) {
int i1[3] = { src_size[0] * src_rank.ix, src_size[1] * src_rank.jy, src_size[2] * src_rank.kz };
int i2[3] = { i1[0] + src_size[0] - 1, i1[1] + src_size[1] - 1, i1[2] + src_size[2] - 1 };
for ( size_t i=0; i<dst_rank.nx; i++ ) {
for ( size_t j=0; j<dst_rank.ny; j++ ) {
for ( size_t k=0; k<dst_rank.nz; k++ ) {
for ( int i=0; i<dst_rank.nx; i++ ) {
for ( int j=0; j<dst_rank.ny; j++ ) {
for ( int k=0; k<dst_rank.nz; k++ ) {
int j1[3] = { i * dst_size[0], j * dst_size[1], k * dst_size[2] };
int j2[3] = { j1[0] + dst_size[0] - 1, j1[1] + dst_size[1] - 1, j1[2] + dst_size[2] - 1 };
auto index = calcOverlap( i1, i2, j1, j2 );
@ -60,14 +61,14 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
}
std::vector<MPI_Request> send_request( send_rank.size() );
for (size_t i=0; i<send_rank.size(); i++)
MPI_Isend( send_data[i].data(), sizeof(TYPE)*send_data[i].length(), MPI_BYTE, send_rank[i], 5462, comm, &send_request[i]);
send_request[i] = comm.Isend( send_data[i].data(), send_data[i].length(), send_rank[i], 5462 );
// Unpack data from the appropriate ranks (including myself)
Array<TYPE> dst_data( dst_size[0], dst_size[1], dst_size[2] );
int i1[3] = { dst_size[0] * dst_rank.ix, dst_size[1] * dst_rank.jy, dst_size[2] * dst_rank.kz };
int i2[3] = { i1[0] + dst_size[0] - 1, i1[1] + dst_size[1] - 1, i1[2] + dst_size[2] - 1 };
for ( size_t i=0; i<src_rank.nx; i++ ) {
for ( size_t j=0; j<src_rank.ny; j++ ) {
for ( size_t k=0; k<src_rank.nz; k++ ) {
for ( int i=0; i<src_rank.nx; i++ ) {
for ( int j=0; j<src_rank.ny; j++ ) {
for ( int k=0; k<src_rank.nz; k++ ) {
int j1[3] = { i * src_size[0], j * src_size[1], k * src_size[2] };
int j2[3] = { j1[0] + src_size[0] - 1, j1[1] + src_size[1] - 1, j1[2] + src_size[2] - 1 };
auto index = calcOverlap( i1, i2, j1, j2 );
@ -75,17 +76,14 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
continue;
int rank = src_rank.getRankForBlock(i,j,k);
Array<TYPE> data( index[1] - index[0] + 1, index[3] - index[2] + 1, index[5] - index[4] + 1 );
MPI_Recv( data.data(), sizeof(TYPE)*data.length(), MPI_BYTE, rank, 5462, comm, MPI_STATUS_IGNORE );
comm.recv( data.data(), data.length(), rank, 5462 );
dst_data.copySubset( index, data );
}
}
}
// Free data
MPI_Waitall( send_request.size(), send_request.data(), MPI_STATUSES_IGNORE );
comm.waitAll( send_request.size(), send_request.data() );
return dst_data;
#else
return src_data.subset( { 0, dst_size[0]-1, 0, dst_size[1]-1, 0, dst_size[2]-1 );
#endif
}
@ -94,27 +92,11 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
* Structure to fill halo cells *
********************************************************/
template<class TYPE>
fillHalo<TYPE>::fillHalo( MPI_Comm comm_, const RankInfoStruct& info_,
fillHalo<TYPE>::fillHalo( const Utilities::MPI& comm_, const RankInfoStruct& info_,
std::array<int,3> n_, std::array<int,3> ng_, int tag0, int depth_,
std::array<bool,3> fill, std::array<bool,3> periodic ):
comm(comm_), info(info_), n(n_), ng(ng_), depth(depth_)
{
if ( std::is_same<TYPE,double>() ) {
N_type = 1;
datatype = MPI_DOUBLE;
} else if ( std::is_same<TYPE,float>() ) {
N_type = 1;
datatype = MPI_FLOAT;
} else if ( sizeof(TYPE)%sizeof(double)==0 ) {
N_type = sizeof(TYPE) / sizeof(double);
datatype = MPI_DOUBLE;
} else if ( sizeof(TYPE)%sizeof(float)==0 ) {
N_type = sizeof(TYPE) / sizeof(float);
datatype = MPI_FLOAT;
} else {
N_type = sizeof(TYPE);
datatype = MPI_BYTE;
}
// Set the fill pattern
memset(fill_pattern,0,sizeof(fill_pattern));
if ( fill[0] ) {
@ -251,8 +233,8 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
for (int k=0; k<3; k++) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Irecv( recv[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype,
info.rank[i][j][k], tag[2-i][2-j][2-k], comm, &recv_req[i][j][k] );
recv_req[i][j][k] = comm.Irecv( recv[i][j][k], depth2*N_send_recv[i][j][k],
info.rank[i][j][k], tag[2-i][2-j][2-k] );
}
}
}
@ -263,19 +245,18 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
if ( !fill_pattern[i][j][k] )
continue;
pack( data, i-1, j-1, k-1, send[i][j][k] );
MPI_Isend( send[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype,
info.rank[i][j][k], tag[i][j][k], comm, &send_req[i][j][k] );
send_req[i][j][k] = comm.Isend( send[i][j][k], depth2*N_send_recv[i][j][k],
info.rank[i][j][k], tag[i][j][k] );
}
}
}
// Recv the dst data and unpack (we recive in reverse order to match the sends)
MPI_Status status;
for (int i=2; i>=0; i--) {
for (int j=2; j>=0; j--) {
for (int k=2; k>=0; k--) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Wait(&recv_req[i][j][k],&status);
comm.wait( recv_req[i][j][k] );
unpack( data, i-1, j-1, k-1, recv[i][j][k] );
}
}
@ -286,7 +267,7 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
for (int k=0; k<3; k++) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Wait(&send_req[i][j][k],&status);
comm.wait( send_req[i][j][k] );
}
}
}

295
common/Domain.cpp
View File

@ -12,7 +12,7 @@
#include "common/Domain.h"
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
// Inline function to read line without a return argument
@ -58,12 +58,14 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL),
id(NULL)
{
NULL_USE( rnk );
NULL_USE( npy );
NULL_USE( npz );
// set up the neighbor ranks
int myrank;
MPI_Comm_rank( Comm, &myrank );
int myrank = Comm.getRank();
rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz );
MPI_Barrier(Comm);
Comm.barrier();
auto db = std::make_shared<Database>( );
db->putScalar<int>( "BC", BC );
@ -73,10 +75,9 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
db->putVector<double>( "L", { lx, ly, lz } );
initialize( db );
}
Domain::Domain( std::shared_ptr<Database> db, MPI_Comm Communicator):
Domain::Domain( std::shared_ptr<Database> db, const Utilities::MPI& Communicator):
database(db), Nx(0), Ny(0), Nz(0),
Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0),
Comm(MPI_COMM_NULL),
inlet_layers_x(0), inlet_layers_y(0), inlet_layers_z(0),
outlet_layers_x(0), outlet_layers_y(0), outlet_layers_z(0),
inlet_layers_phase(1),outlet_layers_phase(2),
@ -106,14 +107,13 @@ Domain::Domain( std::shared_ptr<Database> db, MPI_Comm Communicator):
recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL),
id(NULL)
{
MPI_Comm_dup(Communicator,&Comm);
Comm = Communicator.dup();
// set up the neighbor ranks
int myrank;
MPI_Comm_rank( Comm, &myrank );
int myrank = Comm.getRank();
initialize( db );
rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz );
MPI_Barrier(Comm);
Comm.barrier();
}
Domain::~Domain()
@ -162,10 +162,6 @@ Domain::~Domain()
delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ;
// Free id
delete [] id;
// Free the communicator
if ( Comm != MPI_COMM_WORLD && Comm != MPI_COMM_NULL ) {
MPI_Comm_free(&Comm);
}
}
void Domain::initialize( std::shared_ptr<Database> db )
@ -216,8 +212,7 @@ void Domain::initialize( std::shared_ptr<Database> db )
Ny = ny+2;
Nz = nz+2;
// Initialize ranks
int myrank;
MPI_Comm_rank( Comm, &myrank );
int myrank = Comm.getRank();
rank_info = RankInfoStruct(myrank,nproc[0],nproc[1],nproc[2]);
// inlet layers only apply to lower part of domain
if (rank_info.ix > 0) inlet_layers_x = 0;
@ -236,12 +231,11 @@ void Domain::initialize( std::shared_ptr<Database> db )
id = new signed char[N];
memset(id,0,N);
BoundaryCondition = d_db->getScalar<int>("BC");
int nprocs;
MPI_Comm_size( Comm, &nprocs );
int nprocs = Comm.getSize();
INSIST(nprocs == nproc[0]*nproc[1]*nproc[2],"Fatal error in processor count!");
}
void Domain::Decomp(std::string Filename)
void Domain::Decomp( const std::string& Filename )
{
//.......................................................................
// Reading the domain information file
@ -251,7 +245,6 @@ void Domain::Decomp(std::string Filename)
int nprocs, nprocx, nprocy, nprocz, nx, ny, nz;
int64_t global_Nx,global_Ny,global_Nz;
int64_t i,j,k,n;
int BC=0;
int64_t xStart,yStart,zStart;
int checkerSize;
//int inlet_layers_x, inlet_layers_y, inlet_layers_z;
@ -329,10 +322,9 @@ void Domain::Decomp(std::string Filename)
char *SegData = NULL;
if (RANK==0){
printf("Input media: %s\n",Filename.c_str());
printf("Relabeling %lu values\n",ReadValues.size());
for (int idx=0; idx<ReadValues.size(); idx++){
for (size_t idx=0; idx<ReadValues.size(); idx++){
int oldvalue=ReadValues[idx];
int newvalue=WriteValues[idx];
printf("oldvalue=%d, newvalue =%d \n",oldvalue,newvalue);
@ -375,7 +367,7 @@ void Domain::Decomp(std::string Filename)
n = k*global_Nx*global_Ny+j*global_Nx+i;
//char locval = loc_id[n];
char locval = SegData[n];
for (int idx=0; idx<ReadValues.size(); idx++){
for (size_t idx=0; idx<ReadValues.size(); idx++){
signed char oldvalue=ReadValues[idx];
signed char newvalue=WriteValues[idx];
if (locval == oldvalue){
@ -388,7 +380,7 @@ void Domain::Decomp(std::string Filename)
}
}
if (RANK==0){
for (int idx=0; idx<ReadValues.size(); idx++){
for (size_t idx=0; idx<ReadValues.size(); idx++){
long int label=ReadValues[idx];
long int count=LabelCount[idx];
printf("Label=%ld, Count=%ld \n",label,count);
@ -476,7 +468,7 @@ void Domain::Decomp(std::string Filename)
printf("Checkerboard pattern at y outlet for %i layers \n",outlet_layers_y);
// use checkerboard pattern
for (int k = 0; k<global_Nz; k++){
for (int j = yStart + ny*nprocy - outlet_layers_y; i < yStart + ny*nprocy; j++){
for (int j = yStart + ny*nprocy - outlet_layers_y; j < yStart + ny*nprocy; j++){
for (int i = 0; i<global_Nx; i++){
if ( (i/checkerSize + k/checkerSize)%2 == 0){
// void checkers
@ -485,6 +477,7 @@ void Domain::Decomp(std::string Filename)
else{
// solid checkers
SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
}
}
}
@ -567,7 +560,7 @@ void Domain::Decomp(std::string Filename)
}
else{
//printf("Sending data to process %i \n", rnk);
MPI_Send(loc_id,N,MPI_CHAR,rnk,15,Comm);
Comm.send(loc_id,N,rnk,15);
}
// Write the data for this rank data
sprintf(LocalRankFilename,"ID.%05i",rnk+rank_offset);
@ -582,9 +575,9 @@ void Domain::Decomp(std::string Filename)
else{
// Recieve the subdomain from rank = 0
//printf("Ready to recieve data %i at process %i \n", N,rank);
MPI_Recv(id,N,MPI_CHAR,0,15,Comm,MPI_STATUS_IGNORE);
Comm.recv(id,N,0,15);
}
MPI_Barrier(Comm);
Comm.barrier();
// Compute the porosity
double sum;
@ -635,13 +628,13 @@ void Domain::Decomp(std::string Filename)
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,Comm);
sum = Comm.sumReduce(sum_local);
porosity = sum*iVol_global;
if (rank()==0) printf("Media porosity = %f \n",porosity);
//.........................................................
}
void Domain::AggregateLabels(char *FILENAME){
void Domain::AggregateLabels( const std::string& filename ){
int nx = Nx;
int ny = Ny;
@ -677,7 +670,7 @@ void Domain::AggregateLabels(char *FILENAME){
}
}
}
MPI_Barrier(Comm);
Comm.barrier();
// populate the FullID
if (rank() == 0){
@ -703,7 +696,7 @@ void Domain::AggregateLabels(char *FILENAME){
ipx = (rnk - ipz*npx*npy - ipy*npx);
//printf("ipx=%i ipy=%i ipz=%i\n", ipx, ipy, ipz);
int tag = 15+rnk;
MPI_Recv(LocalID,local_size,MPI_CHAR,rnk,tag,Comm,MPI_STATUS_IGNORE);
Comm.recv(LocalID,local_size,rnk,tag);
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){
for (int i=1; i<nx-1; i++){
@ -718,8 +711,7 @@ void Domain::AggregateLabels(char *FILENAME){
}
}
// write the output
FILE *OUTFILE;
OUTFILE = fopen(FILENAME,"wb");
FILE *OUTFILE = fopen(filename.c_str(),"wb");
fwrite(FullID,1,full_size,OUTFILE);
fclose(OUTFILE);
}
@ -727,9 +719,9 @@ void Domain::AggregateLabels(char *FILENAME){
// send LocalID to rank=0
int tag = 15+ rank();
int dstrank = 0;
MPI_Send(LocalID,local_size,MPI_CHAR,dstrank,tag,Comm);
Comm.send(LocalID,local_size,dstrank,tag);
}
MPI_Barrier(Comm);
Comm.barrier();
}
@ -854,45 +846,45 @@ void Domain::CommInit()
sendBuf_YZ = new int [sendCount_YZ];
sendBuf_XZ = new int [sendCount_XZ];
//......................................................................................
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x(),sendtag+0,Comm,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X(),recvtag+0,Comm,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X(),sendtag+1,Comm,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x(),recvtag+1,Comm,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y(),sendtag+2,Comm,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y(),recvtag+2,Comm,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y(),sendtag+3,Comm,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y(),recvtag+3,Comm,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z(),sendtag+4,Comm,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z(),recvtag+4,Comm,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z(),sendtag+5,Comm,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z(),recvtag+5,Comm,&req2[5]);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy(),sendtag+6,Comm,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY(),recvtag+6,Comm,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY(),sendtag+7,Comm,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy(),recvtag+7,Comm,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy(),sendtag+8,Comm,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY(),recvtag+8,Comm,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY(),sendtag+9,Comm,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy(),recvtag+9,Comm,&req2[9]);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz(),sendtag+10,Comm,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ(),recvtag+10,Comm,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ(),sendtag+11,Comm,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz(),recvtag+11,Comm,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz(),sendtag+12,Comm,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ(),recvtag+12,Comm,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ(),sendtag+13,Comm,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz(),recvtag+13,Comm,&req2[13]);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz(),sendtag+14,Comm,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ(),recvtag+14,Comm,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ(),sendtag+15,Comm,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz(),recvtag+15,Comm,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz(),sendtag+16,Comm,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ(),recvtag+16,Comm,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ(),sendtag+17,Comm,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz(),recvtag+17,Comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(Comm);
req1[0] = Comm.Isend(&sendCount_x,1,rank_x(),sendtag+0);
req2[0] = Comm.Irecv(&recvCount_X,1,rank_X(),recvtag+0);
req1[1] = Comm.Isend(&sendCount_X,1,rank_X(),sendtag+1);
req2[1] = Comm.Irecv(&recvCount_x,1,rank_x(),recvtag+1);
req1[2] = Comm.Isend(&sendCount_y,1,rank_y(),sendtag+2);
req2[2] = Comm.Irecv(&recvCount_Y,1,rank_Y(),recvtag+2);
req1[3] = Comm.Isend(&sendCount_Y,1,rank_Y(),sendtag+3);
req2[3] = Comm.Irecv(&recvCount_y,1,rank_y(),recvtag+3);
req1[4] = Comm.Isend(&sendCount_z,1,rank_z(),sendtag+4);
req2[4] = Comm.Irecv(&recvCount_Z,1,rank_Z(),recvtag+4);
req1[5] = Comm.Isend(&sendCount_Z,1,rank_Z(),sendtag+5);
req2[5] = Comm.Irecv(&recvCount_z,1,rank_z(),recvtag+5);
req1[6] = Comm.Isend(&sendCount_xy,1,rank_xy(),sendtag+6);
req2[6] = Comm.Irecv(&recvCount_XY,1,rank_XY(),recvtag+6);
req1[7] = Comm.Isend(&sendCount_XY,1,rank_XY(),sendtag+7);
req2[7] = Comm.Irecv(&recvCount_xy,1,rank_xy(),recvtag+7);
req1[8] = Comm.Isend(&sendCount_Xy,1,rank_Xy(),sendtag+8);
req2[8] = Comm.Irecv(&recvCount_xY,1,rank_xY(),recvtag+8);
req1[9] = Comm.Isend(&sendCount_xY,1,rank_xY(),sendtag+9);
req2[9] = Comm.Irecv(&recvCount_Xy,1,rank_Xy(),recvtag+9);
req1[10] = Comm.Isend(&sendCount_xz,1,rank_xz(),sendtag+10);
req2[10] = Comm.Irecv(&recvCount_XZ,1,rank_XZ(),recvtag+10);
req1[11] = Comm.Isend(&sendCount_XZ,1,rank_XZ(),sendtag+11);
req2[11] = Comm.Irecv(&recvCount_xz,1,rank_xz(),recvtag+11);
req1[12] = Comm.Isend(&sendCount_Xz,1,rank_Xz(),sendtag+12);
req2[12] = Comm.Irecv(&recvCount_xZ,1,rank_xZ(),recvtag+12);
req1[13] = Comm.Isend(&sendCount_xZ,1,rank_xZ(),sendtag+13);
req2[13] = Comm.Irecv(&recvCount_Xz,1,rank_Xz(),recvtag+13);
req1[14] = Comm.Isend(&sendCount_yz,1,rank_yz(),sendtag+14);
req2[14] = Comm.Irecv(&recvCount_YZ,1,rank_YZ(),recvtag+14);
req1[15] = Comm.Isend(&sendCount_YZ,1,rank_YZ(),sendtag+15);
req2[15] = Comm.Irecv(&recvCount_yz,1,rank_yz(),recvtag+15);
req1[16] = Comm.Isend(&sendCount_Yz,1,rank_Yz(),sendtag+16);
req2[16] = Comm.Irecv(&recvCount_yZ,1,rank_yZ(),recvtag+16);
req1[17] = Comm.Isend(&sendCount_yZ,1,rank_yZ(),sendtag+17);
req2[17] = Comm.Irecv(&recvCount_Yz,1,rank_Yz(),recvtag+17);
Comm.waitAll(18,req1);
Comm.waitAll(18,req2);
Comm.barrier();
//......................................................................................
// recv buffers
recvList_x = new int [recvCount_x];
@ -914,44 +906,44 @@ void Domain::CommInit()
recvList_YZ = new int [recvCount_YZ];
recvList_XZ = new int [recvCount_XZ];
//......................................................................................
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x(),sendtag,Comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X(),recvtag,Comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X(),sendtag,Comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x(),recvtag,Comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y(),sendtag,Comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y(),recvtag,Comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y(),sendtag,Comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y(),recvtag,Comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z(),sendtag,Comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z(),recvtag,Comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z(),sendtag,Comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z(),recvtag,Comm,&req2[5]);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy(),sendtag,Comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY(),recvtag,Comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY(),sendtag,Comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy(),recvtag,Comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy(),sendtag,Comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY(),recvtag,Comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY(),sendtag,Comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy(),recvtag,Comm,&req2[9]);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz(),sendtag,Comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ(),recvtag,Comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ(),sendtag,Comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz(),recvtag,Comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz(),sendtag,Comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ(),recvtag,Comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ(),sendtag,Comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz(),recvtag,Comm,&req2[13]);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz(),sendtag,Comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ(),recvtag,Comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ(),sendtag,Comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz(),recvtag,Comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz(),sendtag,Comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ(),recvtag,Comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ(),sendtag,Comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz(),recvtag,Comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
req1[0] = Comm.Isend(sendList_x,sendCount_x,rank_x(),sendtag);
req2[0] = Comm.Irecv(recvList_X,recvCount_X,rank_X(),recvtag);
req1[1] = Comm.Isend(sendList_X,sendCount_X,rank_X(),sendtag);
req2[1] = Comm.Irecv(recvList_x,recvCount_x,rank_x(),recvtag);
req1[2] = Comm.Isend(sendList_y,sendCount_y,rank_y(),sendtag);
req2[2] = Comm.Irecv(recvList_Y,recvCount_Y,rank_Y(),recvtag);
req1[3] = Comm.Isend(sendList_Y,sendCount_Y,rank_Y(),sendtag);
req2[3] = Comm.Irecv(recvList_y,recvCount_y,rank_y(),recvtag);
req1[4] = Comm.Isend(sendList_z,sendCount_z,rank_z(),sendtag);
req2[4] = Comm.Irecv(recvList_Z,recvCount_Z,rank_Z(),recvtag);
req1[5] = Comm.Isend(sendList_Z,sendCount_Z,rank_Z(),sendtag);
req2[5] = Comm.Irecv(recvList_z,recvCount_z,rank_z(),recvtag);
req1[6] = Comm.Isend(sendList_xy,sendCount_xy,rank_xy(),sendtag);
req2[6] = Comm.Irecv(recvList_XY,recvCount_XY,rank_XY(),recvtag);
req1[7] = Comm.Isend(sendList_XY,sendCount_XY,rank_XY(),sendtag);
req2[7] = Comm.Irecv(recvList_xy,recvCount_xy,rank_xy(),recvtag);
req1[8] = Comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy(),sendtag);
req2[8] = Comm.Irecv(recvList_xY,recvCount_xY,rank_xY(),recvtag);
req1[9] = Comm.Isend(sendList_xY,sendCount_xY,rank_xY(),sendtag);
req2[9] = Comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy(),recvtag);
req1[10] = Comm.Isend(sendList_xz,sendCount_xz,rank_xz(),sendtag);
req2[10] = Comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ(),recvtag);
req1[11] = Comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ(),sendtag);
req2[11] = Comm.Irecv(recvList_xz,recvCount_xz,rank_xz(),recvtag);
req1[12] = Comm.Isend(sendList_Xz,sendCount_Xz,rank_Xz(),sendtag);
req2[12] = Comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ(),recvtag);
req1[13] = Comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ(),sendtag);
req2[13] = Comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz(),recvtag);
req1[14] = Comm.Isend(sendList_yz,sendCount_yz,rank_yz(),sendtag);
req2[14] = Comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ(),recvtag);
req1[15] = Comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ(),sendtag);
req2[15] = Comm.Irecv(recvList_yz,recvCount_yz,rank_yz(),recvtag);
req1[16] = Comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz(),sendtag);
req2[16] = Comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ(),recvtag);
req1[17] = Comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ(),sendtag);
req2[17] = Comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz(),recvtag);
Comm.waitAll(18,req1);
Comm.waitAll(18,req2);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -1094,7 +1086,7 @@ void Domain::ReadIDs(){
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,Comm);
sum = Comm.sumReduce(sum_local);
porosity = sum*iVol_global;
if (rank()==0) printf("Media porosity = %f \n",porosity);
//.........................................................
@ -1141,42 +1133,24 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh)
PackMeshData(sendList_yZ, sendCount_yZ ,sendData_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendData_YZ, MeshData);
//......................................................................................
MPI_Sendrecv(sendData_x,sendCount_x,MPI_DOUBLE,rank_x(),sendtag,
recvData_X,recvCount_X,MPI_DOUBLE,rank_X(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_X,sendCount_X,MPI_DOUBLE,rank_X(),sendtag,
recvData_x,recvCount_x,MPI_DOUBLE,rank_x(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_y,sendCount_y,MPI_DOUBLE,rank_y(),sendtag,
recvData_Y,recvCount_Y,MPI_DOUBLE,rank_Y(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Y,sendCount_Y,MPI_DOUBLE,rank_Y(),sendtag,
recvData_y,recvCount_y,MPI_DOUBLE,rank_y(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_z,sendCount_z,MPI_DOUBLE,rank_z(),sendtag,
recvData_Z,recvCount_Z,MPI_DOUBLE,rank_Z(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Z,sendCount_Z,MPI_DOUBLE,rank_Z(),sendtag,
recvData_z,recvCount_z,MPI_DOUBLE,rank_z(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_xy,sendCount_xy,MPI_DOUBLE,rank_xy(),sendtag,
recvData_XY,recvCount_XY,MPI_DOUBLE,rank_XY(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_XY,sendCount_XY,MPI_DOUBLE,rank_XY(),sendtag,
recvData_xy,recvCount_xy,MPI_DOUBLE,rank_xy(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy(),sendtag,
recvData_xY,recvCount_xY,MPI_DOUBLE,rank_xY(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_xY,sendCount_xY,MPI_DOUBLE,rank_xY(),sendtag,
recvData_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_xz,sendCount_xz,MPI_DOUBLE,rank_xz(),sendtag,
recvData_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ(),sendtag,
recvData_xz,recvCount_xz,MPI_DOUBLE,rank_xz(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz(),sendtag,
recvData_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ(),sendtag,
recvData_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_yz,sendCount_yz,MPI_DOUBLE,rank_yz(),sendtag,
recvData_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ(),sendtag,
recvData_yz,recvCount_yz,MPI_DOUBLE,rank_yz(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz(),sendtag,
recvData_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ(),recvtag,Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ(),sendtag,
recvData_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz(),recvtag,Comm,MPI_STATUS_IGNORE);
Comm.sendrecv(sendData_x,sendCount_x,rank_x(),sendtag,recvData_X,recvCount_X,rank_X(),recvtag);
Comm.sendrecv(sendData_X,sendCount_X,rank_X(),sendtag,recvData_x,recvCount_x,rank_x(),recvtag);
Comm.sendrecv(sendData_y,sendCount_y,rank_y(),sendtag,recvData_Y,recvCount_Y,rank_Y(),recvtag);
Comm.sendrecv(sendData_Y,sendCount_Y,rank_Y(),sendtag,recvData_y,recvCount_y,rank_y(),recvtag);
Comm.sendrecv(sendData_z,sendCount_z,rank_z(),sendtag,recvData_Z,recvCount_Z,rank_Z(),recvtag);
Comm.sendrecv(sendData_Z,sendCount_Z,rank_Z(),sendtag,recvData_z,recvCount_z,rank_z(),recvtag);
Comm.sendrecv(sendData_xy,sendCount_xy,rank_xy(),sendtag,recvData_XY,recvCount_XY,rank_XY(),recvtag);
Comm.sendrecv(sendData_XY,sendCount_XY,rank_XY(),sendtag,recvData_xy,recvCount_xy,rank_xy(),recvtag);
Comm.sendrecv(sendData_Xy,sendCount_Xy,rank_Xy(),sendtag,recvData_xY,recvCount_xY,rank_xY(),recvtag);
Comm.sendrecv(sendData_xY,sendCount_xY,rank_xY(),sendtag,recvData_Xy,recvCount_Xy,rank_Xy(),recvtag);
Comm.sendrecv(sendData_xz,sendCount_xz,rank_xz(),sendtag,recvData_XZ,recvCount_XZ,rank_XZ(),recvtag);
Comm.sendrecv(sendData_XZ,sendCount_XZ,rank_XZ(),sendtag,recvData_xz,recvCount_xz,rank_xz(),recvtag);
Comm.sendrecv(sendData_Xz,sendCount_Xz,rank_Xz(),sendtag,recvData_xZ,recvCount_xZ,rank_xZ(),recvtag);
Comm.sendrecv(sendData_xZ,sendCount_xZ,rank_xZ(),sendtag,recvData_Xz,recvCount_Xz,rank_Xz(),recvtag);
Comm.sendrecv(sendData_yz,sendCount_yz,rank_yz(),sendtag,recvData_YZ,recvCount_YZ,rank_YZ(),recvtag);
Comm.sendrecv(sendData_YZ,sendCount_YZ,rank_YZ(),sendtag,recvData_yz,recvCount_yz,rank_yz(),recvtag);
Comm.sendrecv(sendData_Yz,sendCount_Yz,rank_Yz(),sendtag,recvData_yZ,recvCount_yZ,rank_yZ(),recvtag);
Comm.sendrecv(sendData_yZ,sendCount_yZ,rank_yZ(),sendtag,recvData_Yz,recvCount_Yz,rank_Yz(),recvtag);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvData_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvData_X, MeshData);
@ -1199,19 +1173,18 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh)
}
// Ideally stuff below here should be moved somewhere else -- doesn't really belong here
void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np)
void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, size_t Np)
{
int q,n;
double value;
ofstream File(FILENAME,ios::binary);
for (n=0; n<Np; n++){
for (size_t n=0; n<Np; n++){
// Write the two density values
value = cDen[n];
File.write((char*) &value, sizeof(value));
value = cDen[Np+n];
File.write((char*) &value, sizeof(value));
// Write the even distributions
for (q=0; q<19; q++){
for (size_t q=0; q<19; q++){
value = cfq[q*Np+n];
File.write((char*) &value, sizeof(value));
}
@ -1220,16 +1193,15 @@ void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq
}
void ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, int Np)
void ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, size_t Np)
{
int q=0, n=0;
double value=0;
ifstream File(FILENAME,ios::binary);
for (n=0; n<Np; n++){
for (size_t n=0; n<Np; n++){
File.read((char*) &value, sizeof(value));
cPhi[n] = value;
// Read the distributions
for (q=0; q<19; q++){
for (size_t q=0; q<19; q++){
File.read((char*) &value, sizeof(value));
cfq[q*Np+n] = value;
}
@ -1237,13 +1209,12 @@ void ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, int Np)
File.close();
}
void ReadBinaryFile(char *FILENAME, double *Data, int N)
void ReadBinaryFile(char *FILENAME, double *Data, size_t N)
{
int n;
double value;
ifstream File(FILENAME,ios::binary);
if (File.good()){
for (n=0; n<N; n++){
for (size_t n=0; n<N; n++){
// Write the two density values
File.read((char*) &value, sizeof(value));
Data[n] = value;
@ -1251,7 +1222,7 @@ void ReadBinaryFile(char *FILENAME, double *Data, int N)
}
}
else {
for (n=0; n<N; n++) Data[n] = 1.2e-34;
for (size_t n=0; n<N; n++) Data[n] = 1.2e-34;
}
File.close();
}

20
common/Domain.h
View File

@ -12,11 +12,10 @@
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/Database.h"
class Domain;
template<class TYPE> class PatchData;
@ -64,7 +63,7 @@ private:
class Domain{
public:
//! Default constructor
Domain( std::shared_ptr<Database> db, MPI_Comm Communicator);
Domain( std::shared_ptr<Database> db, const Utilities::MPI& Communicator);
//! Obsolete constructor
Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
@ -117,12 +116,10 @@ public: // Public variables (need to create accessors instead)
double porosity;
RankInfoStruct rank_info;
MPI_Comm Comm; // MPI Communicator for this domain
Utilities::MPI Comm; // MPI Communicator for this domain
int BoundaryCondition;
MPI_Group Group; // Group of processors associated with this domain
//**********************************
// MPI ranks for all 18 neighbors
//**********************************
@ -178,11 +175,11 @@ public: // Public variables (need to create accessors instead)
signed char *id;
void ReadIDs();
void Decomp(std::string Filename);
void Decomp( const std::string& filename );
void CommunicateMeshHalo(DoubleArray &Mesh);
void CommInit();
int PoreCount();
void AggregateLabels(char *FILENAME);
void AggregateLabels( const std::string& filename );
private:
@ -192,7 +189,6 @@ private:
//......................................................................................
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z;
int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ;
@ -245,10 +241,10 @@ private:
};
void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np);
void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, size_t Np);
void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, int Np);
void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, size_t Np);
void ReadBinaryFile(char *FILENAME, double *Data, int N);
void ReadBinaryFile(char *FILENAME, double *Data, size_t N);
#endif

1176
common/MPI.I Normal file
View File

File diff suppressed because it is too large Load Diff

3806
common/MPI.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

1159
common/MPI.h Normal file
View File

File diff suppressed because it is too large Load Diff

266
common/MPI_Helpers.cpp
View File

@ -1,266 +0,0 @@
#include "common/MPI_Helpers.h"
#include "common/Utilities.h"
/********************************************************
* Return the MPI data type *
********************************************************/
template<> MPI_Datatype getMPItype<char>() {
return MPI_CHAR;
}
template<> MPI_Datatype getMPItype<unsigned char>() {
return MPI_UNSIGNED_CHAR;
}
template<> MPI_Datatype getMPItype<int>() {
return MPI_INT;
}
template<> MPI_Datatype getMPItype<long>() {
return MPI_LONG;
}
template<> MPI_Datatype getMPItype<unsigned long>() {
return MPI_UNSIGNED_LONG;
}
template<> MPI_Datatype getMPItype<long long>() {
return MPI_LONG_LONG;
}
template<> MPI_Datatype getMPItype<float>() {
return MPI_FLOAT;
}
template<> MPI_Datatype getMPItype<double>() {
return MPI_DOUBLE;
}
/********************************************************
* Concrete implimentations for packing/unpacking *
********************************************************/
// unsigned char
template<>
size_t packsize<unsigned char>( const unsigned char& )
{
return sizeof(unsigned char);
}
template<>
void pack<unsigned char>( const unsigned char& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(unsigned char));
}
template<>
void unpack<unsigned char>( unsigned char& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(unsigned char));
}
// char
template<>
size_t packsize<char>( const char& )
{
return sizeof(char);
}
template<>
void pack<char>( const char& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(char));
}
template<>
void unpack<char>( char& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(char));
}
// int
template<>
size_t packsize<int>( const int& )
{
return sizeof(int);
}
template<>
void pack<int>( const int& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(int));
}
template<>
void unpack<int>( int& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(int));
}
// unsigned int
template<>
size_t packsize<unsigned int>( const unsigned int& )
{
return sizeof(unsigned int);
}
template<>
void pack<unsigned int>( const unsigned int& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(int));
}
template<>
void unpack<unsigned int>( unsigned int& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(int));
}
// size_t
template<>
size_t packsize<size_t>( const size_t& )
{
return sizeof(size_t);
}
template<>
void pack<size_t>( const size_t& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(size_t));
}
template<>
void unpack<size_t>( size_t& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(size_t));
}
// std::string
template<>
size_t packsize<std::string>( const std::string& rhs )
{
return rhs.size()+1;
}
template<>
void pack<std::string>( const std::string& rhs, char *buffer )
{
memcpy(buffer,rhs.c_str(),rhs.size()+1);
}
template<>
void unpack<std::string>( std::string& data, const char *buffer )
{
data = std::string(buffer);
}
/********************************************************
* Fake MPI routines *
********************************************************/
#ifndef USE_MPI
int MPI_Init(int*,char***)
{
return 0;
}
int MPI_Init_thread(int*,char***, int required, int *provided )
{
*provided = required;
return 0;
}
int MPI_Finalize()
{
return 0;
}
int MPI_Comm_size( MPI_Comm, int *size )
{
*size = 1;
return 0;
}
int MPI_Comm_rank( MPI_Comm, int *rank )
{
*rank = 0;
return 0;
}
int MPI_Barrier( MPI_Comm )
{
return 0;
}
int MPI_Waitall( int, MPI_Request[], MPI_Status[] )
{
return 0;
}
int MPI_Wait( MPI_Request*, MPI_Status* )
{
return 0;
}
int MPI_Bcast( void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm )
{
return 0;
}
int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
MPI_Comm comm, MPI_Status *status)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm, MPI_Request *request)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
int tag, MPI_Comm comm, MPI_Request *request)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, const int *recvcounts, const int *displs,
MPI_Datatype recvtype, MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
int dest, int sendtag,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
int source, int recvtag,
MPI_Comm comm, MPI_Status *status)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
MPI_Op op, int root, MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Comm_group(MPI_Comm comm, MPI_Group *group)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
{
*newcomm = comm;
return 0;
}
double MPI_Wtime( void )
{
return 0.0;
}
int MPI_Comm_free(MPI_Comm *group)
{
return 0;
}
int MPI_Group_free(MPI_Group *group)
{
return 0;
}
#endif

233
common/MPI_Helpers.h
View File

@ -1,233 +0,0 @@
// This file contains wrappers for MPI routines and functions to pack/unpack data structures
#ifndef MPI_WRAPPERS_INC
#define MPI_WRAPPERS_INC
#include <string.h>
#include <vector>
#include <set>
#include <map>
#ifdef USE_MPI
// Inlcude MPI
#include "mpi.h"
#else
// Create fake MPI types
typedef int MPI_Comm;
typedef int MPI_Request;
typedef int MPI_Status;
#define MPI_COMM_WORLD 0
#define MPI_COMM_SELF 0
#define MPI_COMM_NULL -1
#define MPI_GROUP_NULL -2
#define MPI_STATUS_IGNORE NULL
enum MPI_Datatype { MPI_LOGICAL, MPI_CHAR, MPI_UNSIGNED_CHAR, MPI_INT,
MPI_UNSIGNED, MPI_LONG, MPI_UNSIGNED_LONG, MPI_LONG_LONG, MPI_FLOAT, MPI_DOUBLE };
enum MPI_Op { MPI_MIN, MPI_MAX, MPI_SUM };
typedef int MPI_Group;
#define MPI_THREAD_SINGLE 0
#define MPI_THREAD_FUNNELED 1
#define MPI_THREAD_SERIALIZED 2
#define MPI_THREAD_MULTIPLE 3
// Fake MPI functions
int MPI_Init(int*,char***);
int MPI_Init_thread( int *argc, char ***argv, int required, int *provided );
int MPI_Finalize();
int MPI_Comm_size( MPI_Comm, int *size );
int MPI_Comm_rank( MPI_Comm, int *rank );
int MPI_Barrier(MPI_Comm);
int MPI_Wait(MPI_Request*,MPI_Status*);
int MPI_Waitall(int,MPI_Request[],MPI_Status[]);
int MPI_Bcast(void*,int,MPI_Datatype,int,MPI_Comm);
int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm);
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
MPI_Comm comm, MPI_Status *status);
int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm, MPI_Request *request);
int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
int tag, MPI_Comm comm, MPI_Request *request);
int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
MPI_Comm comm);
int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, const int *recvcounts, const int *displs,
MPI_Datatype recvtype, MPI_Comm comm);
int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
int dest, int sendtag,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
int source, int recvtag,
MPI_Comm comm, MPI_Status *status);
int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
MPI_Op op, int root, MPI_Comm comm);
double MPI_Wtime( void );
int MPI_Comm_group(MPI_Comm comm, MPI_Group *group);
int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm);
int MPI_Comm_free(MPI_Comm *group);
int MPI_Group_free(MPI_Group *group);
int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
#endif
//! Get the size of the MPI_Comm
// Note: this is a thread and interrupt safe function
inline int comm_size( MPI_Comm comm ) {
int size = 1;
MPI_Comm_size( comm, &size );
return size;
}
//! Get the rank of the MPI_Comm
// Note: this is a thread and interrupt safe function
inline int comm_rank( MPI_Comm comm ) {
int rank = 1;
MPI_Comm_rank( comm, &rank );
return rank;
}
//! Get the size of MPI_COMM_WORLD
inline int MPI_WORLD_SIZE( ) {
return comm_size( MPI_COMM_WORLD );
}
//! Get the size of MPI_COMM_WORLD
inline int MPI_WORLD_RANK( ) {
return comm_rank( MPI_COMM_WORLD );
}
//! Return the appropriate MPI datatype for a class
template<class TYPE>
MPI_Datatype getMPItype();
//! Template function to return the buffer size required to pack a class
template<class TYPE>
size_t packsize( const TYPE& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const TYPE& rhs, char *buffer );
//! Template function to unpack a class from a buffer
template<class TYPE>
void unpack( TYPE& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::vector
template<class TYPE>
size_t packsize( const std::vector<TYPE>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::vector<TYPE>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::vector<TYPE>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::pair
template<class TYPE1, class TYPE2>
size_t packsize( const std::pair<TYPE1,TYPE2>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::map
template<class TYPE1, class TYPE2>
size_t packsize( const std::map<TYPE1,TYPE2>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::set
template<class TYPE>
size_t packsize( const std::set<TYPE>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::set<TYPE>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::set<TYPE>& data, const char *buffer );
// Helper functions
inline double sumReduce( MPI_Comm comm, double x )
{
double y = 0;
MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,comm);
return y;
}
inline float sumReduce( MPI_Comm comm, float x )
{
float y = 0;
MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_SUM,comm);
return y;
}
inline int sumReduce( MPI_Comm comm, int x )
{
int y = 0;
MPI_Allreduce(&x,&y,1,MPI_INT,MPI_SUM,comm);
return y;
}
inline bool sumReduce( MPI_Comm comm, bool x )
{
int y = sumReduce( comm, x?1:0 );
return y>0;
}
inline std::vector<float> sumReduce( MPI_Comm comm, const std::vector<float>& x )
{
auto y = x;
MPI_Allreduce(x.data(),y.data(),x.size(),MPI_FLOAT,MPI_SUM,comm);
return y;
}
inline std::vector<int> sumReduce( MPI_Comm comm, const std::vector<int>& x )
{
auto y = x;
MPI_Allreduce(x.data(),y.data(),x.size(),MPI_INT,MPI_SUM,comm);
return y;
}
inline double maxReduce( MPI_Comm comm, double x )
{
double y = 0;
MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_MAX,comm);
return y;
}
inline float maxReduce( MPI_Comm comm, float x )
{
float y = 0;
MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_MAX,comm);
return y;
}
inline int maxReduce( MPI_Comm comm, int x )
{
int y = 0;
MPI_Allreduce(&x,&y,1,MPI_INT,MPI_MAX,comm);
return y;
}
#endif
#include "common/MPI_Helpers.hpp"

44
common/ReadMicroCT.cpp
View File

@ -64,11 +64,11 @@ Array<uint8_t> readMicroCT( const std::string& filename )
// Read the compressed micro CT data and distribute
Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm )
Array<uint8_t> readMicroCT( const Database& domain, const Utilities::MPI& comm )
{
// Get the local problem info
auto n = domain.getVector<int>( "n" );
int rank = comm_rank(MPI_COMM_WORLD);
int rank = comm.getRank();
auto nproc = domain.getVector<int>( "nproc" );
RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] );
@ -81,22 +81,38 @@ Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm )
Array<uint8_t> data;
RankInfoStruct srcRankInfo( rank, Nfx, Nfy, Nfz );
if ( srcRankInfo.ix >= 0 ) {
auto filename = domain.getScalar<std::string>( "Filename" );
char tmp[100];
if ( filename.find( "0x_0y_0z.gbd.gz" ) != std::string::npos ) {
sprintf( tmp, "%ix_%iy_%iz.gbd.gz", srcRankInfo.ix, srcRankInfo.jy, srcRankInfo.kz );
filename = filename.replace( filename.find( "0x_0y_0z.gbd.gz" ), 15, std::string( tmp ) );
} else if ( filename.find( "x0_y0_z0.gbd.gz" ) != std::string::npos ) {
sprintf( tmp, "x%i_y%i_z%i.gbd.gz", srcRankInfo.ix, srcRankInfo.jy, srcRankInfo.kz );
filename = filename.replace( filename.find( "x0_y0_z0.gbd.gz" ), 15, std::string( tmp ) );
} else {
ERROR( "Invalid name for first file" );
}
data = readMicroCT( filename );
auto filename = domain.getScalar<std::string>( "Filename" );
char tmp[100];
if ( filename.find( "0x_0y_0z.gbd.gz" ) != std::string::npos ) {
sprintf( tmp, "%ix_%iy_%iz.gbd.gz", srcRankInfo.ix, srcRankInfo.jy, srcRankInfo.kz );
filename = filename.replace( filename.find( "0x_0y_0z.gbd.gz" ), 15, std::string( tmp ) );
} else if ( filename.find( "x0_y0_z0.gbd.gz" ) != std::string::npos ) {
sprintf( tmp, "x%i_y%i_z%i.gbd.gz", srcRankInfo.ix, srcRankInfo.jy, srcRankInfo.kz );
filename = filename.replace( filename.find( "x0_y0_z0.gbd.gz" ), 15, std::string( tmp ) );
} else {
ERROR( "Invalid name for first file" );
}
data = readMicroCT( filename );
}
// Redistribute the data
data = redistribute( srcRankInfo, data, rankInfo, { n[0], n[1], n[2] }, comm );
// Relabel the data
auto ReadValues = domain.getVector<int>( "ReadValues" );
auto WriteValues = domain.getVector<int>( "WriteValues" );
ASSERT( ReadValues.size() == WriteValues.size() );
int readMaxValue = 0;
for ( auto v : ReadValues )
readMaxValue = std::max( data.max()+1, v );
std::vector<int> map( readMaxValue + 1, -1 );
for ( size_t i=0; i<ReadValues.size(); i++ )
map[ReadValues[i]] = WriteValues[i];
for ( size_t i=0; i<data.length(); i++ ) {
int t = data(i);
ASSERT( t >= 0 && t <= readMaxValue );
data(i) = map[t];
}
return data;
}

3
common/ReadMicroCT.h
View File

@ -5,11 +5,12 @@
#include "common/Array.h"
#include "common/Communication.h"
#include "common/Database.h"
#include "common/MPI.h"
Array<uint8_t> readMicroCT( const std::string& filename );
Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm );
Array<uint8_t> readMicroCT( const Database& domain, const Utilities::MPI& comm );
#endif

251
common/ScaLBL.cpp
View File

@ -5,9 +5,7 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
Lock=false; // unlock the communicator
//......................................................................................
// Create a separate copy of the communicator for the device
//MPI_Comm_group(Dm->Comm,&Group);
//MPI_Comm_create(Dm->Comm,Group,&MPI_COMM_SCALBL);
MPI_Comm_dup(Dm->Comm,&MPI_COMM_SCALBL);
MPI_COMM_SCALBL = Dm->Comm.dup();
//......................................................................................
// Copy the domain size and communication information directly from Dm
Nx = Dm->Nx;
@ -215,7 +213,7 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm->recvList_Yz,recvCount_Yz*sizeof(int));
//......................................................................................
MPI_Barrier(MPI_COMM_SCALBL);
MPI_COMM_SCALBL.barrier();
//...................................................................................
// Set up the recieve distribution lists
@ -288,7 +286,7 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
//...................................................................................
//......................................................................................
MPI_Barrier(MPI_COMM_SCALBL);
MPI_COMM_SCALBL.barrier();
ScaLBL_DeviceBarrier();
//......................................................................................
SendCount = sendCount_x+sendCount_X+sendCount_y+sendCount_Y+sendCount_z+sendCount_Z+
@ -365,7 +363,7 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
int idx,i,j,k,n;
// Check that Map has size matching sub-domain
if (Map.size(0) != Nx)
if ( (int) Map.size(0) != Nx)
ERROR("ScaLBL_Communicator::MemoryOptimizedLayout: Map array dimensions do not match! \n");
// Initialize Map
@ -869,8 +867,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){
ScaLBL_D3Q19_Pack(12,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,dist,N);
ScaLBL_D3Q19_Pack(14,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,dist,N);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 5*sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 5*recvCount_X,rank_X,recvtag);
//...Packing for X face(1,7,9,11,13)................................
ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,dist,N);
ScaLBL_D3Q19_Pack(7,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,dist,N);
@ -878,8 +876,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){
ScaLBL_D3Q19_Pack(11,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,dist,N);
ScaLBL_D3Q19_Pack(13,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,dist,N);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 5*sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 5*recvCount_x,rank_x,recvtag);
//...Packing for y face(4,8,9,16,18).................................
ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,dist,N);
ScaLBL_D3Q19_Pack(8,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,dist,N);
@ -887,8 +885,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){
ScaLBL_D3Q19_Pack(16,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,dist,N);
ScaLBL_D3Q19_Pack(18,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,dist,N);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 5*sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 5*recvCount_Y,rank_Y,recvtag);
//...Packing for Y face(3,7,10,15,17).................................
ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,dist,N);
ScaLBL_D3Q19_Pack(7,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,dist,N);
@ -896,8 +894,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){
ScaLBL_D3Q19_Pack(15,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,dist,N);
ScaLBL_D3Q19_Pack(17,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,dist,N);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 5*sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 5*recvCount_y,rank_y,recvtag);
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,dist,N);
ScaLBL_D3Q19_Pack(12,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,dist,N);
@ -905,8 +903,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){
ScaLBL_D3Q19_Pack(16,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,dist,N);
ScaLBL_D3Q19_Pack(17,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,dist,N);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 5*sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 5*recvCount_Z,rank_Z,recvtag);
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,dist,N);
@ -915,57 +913,57 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){
ScaLBL_D3Q19_Pack(15,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,dist,N);
ScaLBL_D3Q19_Pack(18,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,dist,N);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 5*sendCount_Z,rank_Z,sendtag);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 5*recvCount_z,rank_z,recvtag);
//...Pack the xy edge (8)................................
ScaLBL_D3Q19_Pack(8,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,dist,N);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_SCALBL,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_SCALBL,&req2[6]);
req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag);
req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag);
//...Pack the Xy edge (9)................................
ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,dist,N);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_SCALBL,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_SCALBL,&req2[8]);
req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag);
req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag);
//...Pack the xY edge (10)................................
ScaLBL_D3Q19_Pack(10,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,dist,N);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_SCALBL,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_SCALBL,&req2[9]);
req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag);
req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag);
//...Pack the XY edge (7)................................
ScaLBL_D3Q19_Pack(7,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,dist,N);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_SCALBL,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_SCALBL,&req2[7]);
req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag);
req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag);
//...Pack the xz edge (12)................................
ScaLBL_D3Q19_Pack(12,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,dist,N);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_SCALBL,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_SCALBL,&req2[10]);
req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag);
req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag);
//...Pack the xZ edge (14)................................
ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,dist,N);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_SCALBL,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_SCALBL,&req2[13]);
req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag);
req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag);
//...Pack the Xz edge (13)................................
ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,dist,N);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_SCALBL,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_SCALBL,&req2[12]);
req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag);
req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag);
//...Pack the XZ edge (11)................................
ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,dist,N);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_SCALBL,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_SCALBL,&req2[11]);
req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag);
req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag);
//...Pack the yz edge (16)................................
ScaLBL_D3Q19_Pack(16,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,dist,N);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_SCALBL,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_SCALBL,&req2[14]);
req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag);
req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag);
//...Pack the yZ edge (18)................................
ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,dist,N);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]);
req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag);
req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag);
//...Pack the Yz edge (17)................................
ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,dist,N);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_SCALBL,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]);
req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag);
req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag);
//...Pack the YZ edge (15)................................
ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,dist,N);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_SCALBL,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_SCALBL,&req2[15]);
req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag);
req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag);
//...................................................................................
}
@ -975,8 +973,8 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){
// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_COMM_SCALBL.waitAll(18,req1);
MPI_COMM_SCALBL.waitAll(18,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
@ -1059,8 +1057,8 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){
// Recieves halo and incorporates into D3Q19 based stencil gradient computation
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_COMM_SCALBL.waitAll(18,req1);
MPI_COMM_SCALBL.waitAll(18,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
@ -1153,36 +1151,36 @@ void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){
ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,Aq,N);
ScaLBL_D3Q19_Pack(2,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,Bq,N);
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]);
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 2*sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 2*recvCount_X,rank_X,recvtag);
//...Packing for X face(1,7,9,11,13)................................
ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,Aq,N);
ScaLBL_D3Q19_Pack(1,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,Bq,N);
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]);
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 2*sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 2*recvCount_x,rank_x,recvtag);
//...Packing for y face(4,8,9,16,18).................................
ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,Aq,N);
ScaLBL_D3Q19_Pack(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,Bq,N);
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]);
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 2*sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 2*recvCount_Y,rank_Y,recvtag);
//...Packing for Y face(3,7,10,15,17).................................
ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,Aq,N);
ScaLBL_D3Q19_Pack(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N);
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]);
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 2*sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 2*recvCount_y,rank_y,recvtag);
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,Aq,N);
ScaLBL_D3Q19_Pack(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,Bq,N);
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]);
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 2*sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 2*recvCount_Z,rank_Z,recvtag);
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,Aq,N);
@ -1190,8 +1188,8 @@ void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){
//...................................................................................
// Send all the distributions
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]);
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 2*sendCount_Z,rank_Z,sendtag);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 2*recvCount_z,rank_z,recvtag);
}
@ -1201,8 +1199,8 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){
// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
MPI_COMM_SCALBL.waitAll(6,req1);
MPI_COMM_SCALBL.waitAll(6,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
@ -1293,18 +1291,18 @@ void ScaLBL_Communicator::TriSendD3Q7AA(double *Aq, double *Bq, double *Cq){
//...................................................................................
// Send all the distributions
MPI_Isend(sendbuf_x, 3*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]);
MPI_Irecv(recvbuf_X, 3*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]);
MPI_Isend(sendbuf_X, 3*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]);
MPI_Irecv(recvbuf_x, 3*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]);
MPI_Isend(sendbuf_y, 3*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]);
MPI_Irecv(recvbuf_Y, 3*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]);
MPI_Isend(sendbuf_Y, 3*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]);
MPI_Irecv(recvbuf_y, 3*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]);
MPI_Isend(sendbuf_z, 3*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]);
MPI_Irecv(recvbuf_Z, 3*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]);
MPI_Isend(sendbuf_Z, 3*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]);
MPI_Irecv(recvbuf_z, 3*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 3*sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 3*recvCount_X,rank_X,recvtag);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 3*sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 3*recvCount_x,rank_x,recvtag);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 3*sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 3*recvCount_Y,rank_Y,recvtag);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 3*sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 3*recvCount_y,rank_y,recvtag);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 3*sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 3*recvCount_Z,rank_Z,recvtag);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 3*sendCount_Z,rank_Z,sendtag);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 3*recvCount_z,rank_z,recvtag);
}
@ -1314,8 +1312,8 @@ void ScaLBL_Communicator::TriRecvD3Q7AA(double *Aq, double *Bq, double *Cq){
// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
MPI_COMM_SCALBL.waitAll(6,req1);
MPI_COMM_SCALBL.waitAll(6,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
@ -1409,49 +1407,49 @@ void ScaLBL_Communicator::SendHalo(double *data){
// Send / Recv all the phase indcator field values
//...................................................................................
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_SCALBL,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_SCALBL,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_SCALBL,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_SCALBL,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_SCALBL,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_SCALBL,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_SCALBL,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_SCALBL,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_SCALBL,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_SCALBL,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_SCALBL,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_SCALBL,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_SCALBL,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_SCALBL,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_SCALBL,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_SCALBL,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_SCALBL,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_SCALBL,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_SCALBL,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_SCALBL,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_SCALBL,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag);
req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag);
req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag);
req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag);
req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag);
req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag);
req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag);
req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag);
req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag);
req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag);
req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag);
req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag);
req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag);
req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag);
req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag);
req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag);
req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag);
req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag);
req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag);
req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag);
req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag);
req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag);
req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag);
req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag);
req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag);
//...................................................................................
}
void ScaLBL_Communicator::RecvHalo(double *data){
//...................................................................................
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_COMM_SCALBL.waitAll(18,req1);
MPI_COMM_SCALBL.waitAll(18,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
//...................................................................................
@ -1480,7 +1478,6 @@ void ScaLBL_Communicator::RecvHalo(double *data){
void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, DoubleArray &regdata){
// Gets data from the device and stores in regular layout
int i,j,k,n,idx;
int Nx = map.size(0);
int Ny = map.size(1);
int Nz = map.size(2);
@ -1492,11 +1489,10 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double
double value;
TmpDat = new double [N];
ScaLBL_CopyToHost(&TmpDat[0],&data[0], N*sizeof(double));
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
n=k*Nx*Ny+j*Nx+i;
idx=map(i,j,k);
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
for (int i=0; i<Nx; i++){
auto idx=map(i,j,k);
if (!(idx<0)){
value=TmpDat[idx];
regdata(i,j,k)=value;
@ -1510,8 +1506,9 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double
}
void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB){
double Value=(vA-vB)/(vA+vB);
void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB)
{
//double Value=(vA-vB)/(vA+vB);
if (kproc == 0) {
// Set the phase indicator field and density on the z inlet
ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N);
@ -1519,8 +1516,9 @@ void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double
}
}
void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB){
double Value=(vA-vB)/(vA+vB);
void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB)
{
//double Value=(vA-vB)/(vA+vB);
if (kproc == nprocz-1){
// Set the phase indicator field and density on the Z outlet
ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N);
@ -1528,7 +1526,8 @@ void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double
}
}
void ScaLBL_Communicator::D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time){
void ScaLBL_Communicator::D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time)
{
//ScaLBL_D3Q19_Pressure_BC_z(int *LIST,fq,din,Nx,Ny,Nz);
if (kproc == 0) {
if (time%2==0){
@ -1563,7 +1562,7 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl
LocInletArea = double(sendCount_z);
else LocInletArea = 0.f;
MPI_Allreduce(&LocInletArea,&InletArea,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_SCALBL);
InletArea = MPI_COMM_SCALBL.sumReduce( LocInletArea );
//printf("Inlet area = %f \n", InletArea);
// Set the flux BC
@ -1572,7 +1571,7 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl
if (kproc == 0)
locsum = ScaLBL_D3Q19_AAeven_Flux_BC_z(dvcSendList_z, fq, flux, InletArea, sendCount_z, N);
MPI_Allreduce(&locsum,&sum,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_SCALBL);
sum = MPI_COMM_SCALBL.sumReduce( locsum );
din = flux/InletArea + sum;
//if (rank==0) printf("computed din (even) =%f \n",din);
if (kproc == 0)
@ -1582,7 +1581,7 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl
if (kproc == 0)
locsum = ScaLBL_D3Q19_AAodd_Flux_BC_z(neighborList, dvcSendList_z, fq, flux, InletArea, sendCount_z, N);
MPI_Allreduce(&locsum,&sum,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_SCALBL);
sum = MPI_COMM_SCALBL.sumReduce( locsum );
din = flux/InletArea + sum;
//if (rank==0) printf("computed din (odd)=%f \n",din);

9
common/ScaLBL.h
View File

@ -119,11 +119,6 @@ extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double
// BOUNDARY CONDITION ROUTINES
//extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din,
// int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout,
// int Nx, int Ny, int Nz, int outlet);
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int Np);
@ -224,10 +219,8 @@ private:
int sendtag,recvtag;
// Give the object it's own MPI communicator
RankInfoStruct rank_info;
MPI_Group Group; // Group of processors associated with this domain
MPI_Comm MPI_COMM_SCALBL; // MPI Communicator for this domain
Utilities::MPI MPI_COMM_SCALBL; // MPI Communicator for this domain
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//......................................................................................
// MPI ranks for all 18 neighbors
//......................................................................................

1
common/SpherePack.cpp
View File

@ -9,7 +9,6 @@
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
#include "common/Database.h"
#include "common/SpherePack.h"

1
common/SpherePack.h
View File

@ -12,7 +12,6 @@
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
#include "common/Database.h"

211
common/UnitTest.cpp
View File

@ -14,44 +14,49 @@
/********************************************************************
* Constructor/Destructor *
********************************************************************/
UnitTest::UnitTest()
UnitTest::UnitTest() : d_verbose( false ), d_comm( MPI_COMM_SELF )
{
#ifdef USE_MPI
comm = MPI_COMM_WORLD;
#endif
if ( Utilities::MPI::MPI_active() )
d_comm = MPI_COMM_WORLD;
}
UnitTest::~UnitTest() { reset(); }
void UnitTest::reset()
{
mutex.lock();
d_mutex.lock();
// Clear the data forcing a reallocation
std::vector<std::string>().swap( pass_messages );
std::vector<std::string>().swap( fail_messages );
std::vector<std::string>().swap( expected_fail_messages );
mutex.unlock();
std::vector<std::string>().swap( d_pass );
std::vector<std::string>().swap( d_fail );
std::vector<std::string>().swap( d_expected );
d_mutex.unlock();
}
/********************************************************************
* Add a pass, fail, expected failure message in a thread-safe way *
********************************************************************/
void UnitTest::passes( const std::string &in )
void UnitTest::passes( std::string in )
{
mutex.lock();
pass_messages.push_back( in );
mutex.unlock();
d_mutex.lock();
if ( d_verbose )
printf( "UnitTest: %i passes: %s\n", d_comm.getRank(), in.data() );
d_pass.emplace_back( std::move( in ) );
d_mutex.unlock();
}
void UnitTest::failure( const std::string &in )
void UnitTest::failure( std::string in )
{
mutex.lock();
fail_messages.push_back( in );
mutex.unlock();
d_mutex.lock();
if ( d_verbose )
printf( "UnitTest: %i failed: %s\n", d_comm.getRank(), in.data() );
d_fail.emplace_back( std::move( in ) );
d_mutex.unlock();
}
void UnitTest::expected_failure( const std::string &in )
void UnitTest::expected_failure( std::string in )
{
mutex.lock();
expected_fail_messages.push_back( in );
mutex.unlock();
d_mutex.lock();
if ( d_verbose )
printf( "UnitTest: %i expected_failure: %s\n", d_comm.getRank(), in.data() );
d_expected.emplace_back( std::move( in ) );
d_mutex.unlock();
}
@ -59,23 +64,6 @@ void UnitTest::expected_failure( const std::string &in )
* Print a global report *
* Note: only rank 0 will print, all messages will be aggregated *
********************************************************************/
inline std::vector<int> UnitTest::allGather( int value ) const
{
int size = getSize();
std::vector<int> data( size, value );
#ifdef USE_MPI
if ( size > 1 )
MPI_Allgather( &value, 1, MPI_INT, data.data(), 1, MPI_INT, comm );
#endif
return data;
}
inline void UnitTest::barrier() const
{
#ifdef USE_MPI
if ( getSize() > 1 )
MPI_Barrier( comm );
#endif
}
static inline void print_messages( const std::vector<std::vector<std::string>> &messages )
{
if ( messages.size() > 1 ) {
@ -93,28 +81,27 @@ static inline void print_messages( const std::vector<std::vector<std::string>> &
}
void UnitTest::report( const int level0 ) const
{
mutex.lock();
int size = getSize();
int rank = getRank();
d_mutex.lock();
int size = d_comm.getSize();
int rank = d_comm.getRank();
// Give all processors a chance to print any remaining messages
d_comm.barrier();
Utilities::sleep_ms( 10 );
// Broadcast the print level from rank 0
int level = level0;
#ifdef USE_MPI
if ( getSize() > 1 )
MPI_Bcast( &level, 1, MPI_INT, 0, comm );
#endif
int level = d_comm.bcast( level0, 0 );
if ( level < 0 || level > 2 )
ERROR( "Invalid print level" );
// Perform a global all gather to get the number of failures per processor
auto N_pass = allGather( pass_messages.size() );
auto N_fail = allGather( fail_messages.size() );
auto N_expected_fail = allGather( expected_fail_messages.size() );
int N_pass_tot = 0;
int N_fail_tot = 0;
int N_expected_fail_tot = 0;
auto N_pass = d_comm.allGather<int>( d_pass.size() );
auto N_fail = d_comm.allGather<int>( d_fail.size() );
auto N_expected = d_comm.allGather<int>( d_expected.size() );
int N_pass_tot = 0;
int N_fail_tot = 0;
int N_expected_tot = 0;
for ( int i = 0; i < size; i++ ) {
N_pass_tot += N_pass[i];
N_fail_tot += N_fail[i];
N_expected_fail_tot += N_expected_fail[i];
N_expected_tot += N_expected[i];
}
// Send all messages to rank 0 (if needed)
std::vector<std::vector<std::string>> pass_messages_rank( size );
@ -122,13 +109,13 @@ void UnitTest::report( const int level0 ) const
std::vector<std::vector<std::string>> expected_fail_rank( size );
// Get the pass messages
if ( ( level == 1 && N_pass_tot <= 20 ) || level == 2 )
pass_messages_rank = UnitTest::gatherMessages( pass_messages, 1 );
pass_messages_rank = UnitTest::gatherMessages( d_pass, 1 );
// Get the fail messages
if ( level == 1 || level == 2 )
fail_messages_rank = UnitTest::gatherMessages( fail_messages, 2 );
fail_messages_rank = UnitTest::gatherMessages( d_fail, 2 );
// Get the expected_fail messages
if ( ( level == 1 && N_expected_fail_tot <= 50 ) || level == 2 )
expected_fail_rank = UnitTest::gatherMessages( expected_fail_messages, 2 );
if ( ( level == 1 && N_expected_tot <= 50 ) || level == 2 )
expected_fail_rank = UnitTest::gatherMessages( d_expected, 2 );
// Print the results of all messages (only rank 0 will print)
if ( rank == 0 ) {
pout << std::endl;
@ -174,31 +161,31 @@ void UnitTest::report( const int level0 ) const
pout << std::endl;
// Print the tests that expected failed
pout << "Tests expected failed" << std::endl;
if ( level == 0 || ( level == 1 && N_expected_fail_tot > 50 ) ) {
if ( level == 0 || ( level == 1 && N_expected_tot > 50 ) ) {
// We want to print a summary
if ( size > 8 ) {
// Print 1 summary for all processors
printp( " %i tests expected failed (use report level 2 for more detail)\n",
N_expected_fail_tot );
N_expected_tot );
} else {
// Print a summary for each processor
for ( int i = 0; i < size; i++ )
printp( " %i tests expected failed (proc %i) (use report level 2 for more "
"detail)\n",
N_expected_fail[i], i );
N_expected[i], i );
}
} else {
// We want to print all messages
for ( int i = 0; i < size; i++ )
ASSERT( (int) expected_fail_rank[i].size() == N_expected_fail[i] );
ASSERT( (int) expected_fail_rank[i].size() == N_expected[i] );
print_messages( expected_fail_rank );
}
pout << std::endl;
}
// Add a barrier to synchronize all processors (rank 0 is much slower)
barrier();
d_comm.barrier();
Utilities::sleep_ms( 10 ); // Need a brief pause to allow any printing to finish
mutex.unlock();
d_mutex.unlock();
}
@ -208,8 +195,8 @@ void UnitTest::report( const int level0 ) const
std::vector<std::vector<std::string>> UnitTest::gatherMessages(
const std::vector<std::string> &local_messages, int tag ) const
{
const int rank = getRank();
const int size = getSize();
const int rank = d_comm.getRank();
const int size = d_comm.getSize();
std::vector<std::vector<std::string>> messages( size );
if ( rank == 0 ) {
// Rank 0 should receive all messages
@ -233,7 +220,6 @@ std::vector<std::vector<std::string>> UnitTest::gatherMessages(
void UnitTest::pack_message_stream(
const std::vector<std::string> &messages, const int rank, const int tag ) const
{
#ifdef USE_MPI
// Get the size of the messages
auto N_messages = (int) messages.size();
auto *msg_size = new int[N_messages];
@ -254,18 +240,11 @@ void UnitTest::pack_message_stream(
k += msg_size[i];
}
// Send the message stream (using a non-blocking send)
MPI_Request request;
MPI_Isend( data, size_data, MPI_CHAR, rank, tag, comm, &request );
auto request = d_comm.Isend( data, size_data, rank, tag );
// Wait for the communication to send and free the temporary memory
MPI_Status status;
MPI_Wait( &request, &status );
d_comm.wait( request );
delete[] data;
delete[] msg_size;
#else
NULL_USE( messages );
NULL_USE( rank );
NULL_USE( tag );
#endif
}
@ -274,20 +253,15 @@ void UnitTest::pack_message_stream(
********************************************************************/
std::vector<std::string> UnitTest::unpack_message_stream( const int rank, const int tag ) const
{
#ifdef USE_MPI
// Probe the message to get the message size
MPI_Status status;
MPI_Probe( rank, tag, comm, &status );
int size_data = -1;
MPI_Get_count( &status, MPI_BYTE, &size_data );
int size_data = d_comm.probe( rank, tag );
ASSERT( size_data >= 0 );
// Allocate memory to receive the data
auto *data = new char[size_data];
// receive the data (using a non-blocking receive)
MPI_Request request;
MPI_Irecv( data, size_data, MPI_CHAR, rank, tag, comm, &request );
auto request = d_comm.Irecv( data, size_data, rank, tag );
// Wait for the communication to be received
MPI_Wait( &request, &status );
d_comm.wait( request );
// Unpack the message stream
int N_messages = 0;
memcpy( &N_messages, data, sizeof( int ) );
@ -303,77 +277,16 @@ std::vector<std::string> UnitTest::unpack_message_stream( const int rank, const
messages[i] = std::string( &data[k], msg_size[i] );
k += msg_size[i];
}
// Delete the temporary memory
delete[] data;
return messages;
#else
NULL_USE( rank );
NULL_USE( tag );
return std::vector<std::string>();
#endif
}
/********************************************************************
* Other functions *
********************************************************************/
int UnitTest::getRank() const
{
int rank = 0;
#ifdef USE_MPI
int flag = 0;
MPI_Initialized( &flag );
if ( flag )
MPI_Comm_rank( comm, &rank );
#endif
return rank;
}
int UnitTest::getSize() const
{
int size = 1;
#ifdef USE_MPI
int flag = 0;
MPI_Initialized( &flag );
if ( flag )
MPI_Comm_size( comm, &size );
#endif
return size;
}
size_t UnitTest::NumPassGlobal() const
{
size_t num = pass_messages.size();
#ifdef USE_MPI
if ( getSize() > 1 ) {
auto send = static_cast<int>( num );
int sum = 0;
MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm );
num = static_cast<size_t>( sum );
}
#endif
return num;
}
size_t UnitTest::NumFailGlobal() const
{
size_t num = fail_messages.size();
#ifdef USE_MPI
if ( getSize() > 1 ) {
auto send = static_cast<int>( num );
int sum = 0;
MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm );
num = static_cast<size_t>( sum );
}
#endif
return num;
}
size_t UnitTest::NumExpectedFailGlobal() const
{
size_t num = expected_fail_messages.size();
#ifdef USE_MPI
if ( getSize() > 1 ) {
auto send = static_cast<int>( num );
int sum = 0;
MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm );
num = static_cast<size_t>( sum );
}
#endif
return num;
}
size_t UnitTest::NumPassGlobal() const { return d_comm.sumReduce( d_pass.size() ); }
size_t UnitTest::NumFailGlobal() const { return d_comm.sumReduce( d_fail.size() ); }
size_t UnitTest::NumExpectedFailGlobal() const { return d_comm.sumReduce( d_expected.size() ); }

71
common/UnitTest.h
View File

@ -1,13 +1,11 @@
#ifndef included_UnitTest
#define included_UnitTest
#include "common/MPI.h"
#include <mutex>
#include <sstream>
#include <string>
#include <vector>
#ifdef USE_MPI
#include "mpi.h"
#endif
/*!
@ -28,47 +26,47 @@
* \endcode
*/
class UnitTest
class UnitTest final
{
public:
//! Constructor
UnitTest();
//! Destructor
virtual ~UnitTest();
~UnitTest();
// Copy constructor
UnitTest( const UnitTest & ) = delete;
// Assignment operator
UnitTest &operator=( const UnitTest & ) = delete;
//! Indicate a passed test (thread-safe)
virtual void passes( const std::string &in );
void passes( std::string in );
//! Indicate a failed test (thread-safe)
virtual void failure( const std::string &in );
void failure( std::string in );
//! Indicate an expected failed test (thread-safe)
virtual void expected_failure( const std::string &in );
void expected_failure( std::string in );
//! Return the number of passed tests locally
virtual size_t NumPassLocal() const { return pass_messages.size(); }
inline size_t NumPassLocal() const { return d_pass.size(); }
//! Return the number of failed tests locally
virtual size_t NumFailLocal() const { return fail_messages.size(); }
inline size_t NumFailLocal() const { return d_fail.size(); }
//! Return the number of expected failed tests locally
virtual size_t NumExpectedFailLocal() const { return expected_fail_messages.size(); }
inline size_t NumExpectedFailLocal() const { return d_expected.size(); }
//! Return the number of passed tests locally
virtual size_t NumPassGlobal() const;
size_t NumPassGlobal() const;
//! Return the number of failed tests locally
virtual size_t NumFailGlobal() const;
size_t NumFailGlobal() const;
//! Return the number of expected failed tests locally
virtual size_t NumExpectedFailGlobal() const;
//! Return the rank of the current processor
int getRank() const;
//! Return the number of processors
int getSize() const;
size_t NumExpectedFailGlobal() const;
/*!
* Print a report of the passed and failed tests.
@ -77,29 +75,28 @@ public:
* to print correctly).
* @param level Optional integer specifying the level of reporting (default: 1)
* 0: Report the number of tests passed, failed, and expected failures.
* 1: Report the number of passed tests (if <=20) or the number passed
* otherwise, report all failures, report the number of expected
* failed tests (if <=50) or the number passed otherwise.
* 1: Report the passed tests (if <=20) or number passed,
* Report all failures,
* Report the expected failed tests (if <=50) or the number passed.
* 2: Report all passed, failed, and expected failed tests.
*/
virtual void report( const int level = 1 ) const;
void report( const int level = 1 ) const;
//! Clear the messages
void reset();
protected:
std::vector<std::string> pass_messages;
std::vector<std::string> fail_messages;
std::vector<std::string> expected_fail_messages;
mutable std::mutex mutex;
#ifdef USE_MPI
MPI_Comm comm;
#endif
//! Make the unit test operator verbose?
void verbose( bool verbose = true ) { d_verbose = verbose; }
private:
// Make the copy constructor private
UnitTest( const UnitTest & ) {}
std::vector<std::string> d_pass;
std::vector<std::string> d_fail;
std::vector<std::string> d_expected;
bool d_verbose;
mutable std::mutex d_mutex;
Utilities::MPI d_comm;
private:
// Function to pack the messages into a single data stream and send to the given processor
// Note: This function does not return until the message stream has been sent
void pack_message_stream(
@ -109,9 +106,7 @@ private:
// Note: This function does not return until the message stream has been received
std::vector<std::string> unpack_message_stream( const int rank, const int tag ) const;
// Helper functions
inline void barrier() const;
inline std::vector<int> allGather( int value ) const;
// Gather the messages
inline std::vector<std::vector<std::string>> gatherMessages(
const std::vector<std::string> &local_messages, int tag ) const;
};

130
common/Utilities.cpp
View File

@ -1,10 +1,130 @@
#include "common/Utilities.h"
#include "StackTrace/StackTrace.h"
#include "StackTrace/ErrorHandlers.h"
#ifdef USE_TIMER
#include "MemoryApp.h"
#include "ProfilerApp.h"
#endif
#ifdef USE_MPI
#include "common/MPI.h"
#endif
#include <math.h>
#include <algorithm>
#include <math.h>
#include <mutex>
// Factor a number into it's prime factors
// OS specific includes / definitions
// clang-format off
#if defined( WIN32 ) || defined( _WIN32 ) || defined( WIN64 ) || defined( _WIN64 )
#define USE_WINDOWS
#elif defined( __APPLE__ )
#define USE_MAC
#elif defined( __linux ) || defined( __linux__ ) || defined( __unix ) || defined( __posix )
#define USE_LINUX
#else
#error Unknown OS
#endif
// clang-format on
// Mutex for Utility functions
static std::mutex Utilities_mutex;
/****************************************************************************
* Function to perform the default startup/shutdown sequences *
****************************************************************************/
void Utilities::startup( int argc, char **argv )
{
NULL_USE( argc );
NULL_USE( argv );
// Disable OpenMP
Utilities::setenv( "OMP_NUM_THREADS", "1" );
Utilities::setenv( "MKL_NUM_THREADS", "1" );
// Start MPI
#ifdef USE_MPI
int provided;
MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
if ( provided < MPI_THREAD_MULTIPLE ) {
int rank;
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
if ( rank == 0 )
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
}
StackTrace::globalCallStackInitialize( MPI_COMM_WORLD );
#endif
// Set the error handlers
Utilities::setAbortBehavior( true, 3 );
Utilities::setErrorHandlers();
}
void Utilities::shutdown()
{
// Clear the error handlers
Utilities::clearErrorHandlers();
StackTrace::clearSignals();
StackTrace::clearSymbols();
int rank = 0;
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
StackTrace::globalCallStackFinalize();
MPI_Barrier( MPI_COMM_WORLD );
MPI_Finalize();
#endif
#ifdef USE_TIMER
PROFILE_DISABLE();
auto memory = MemoryApp::getMemoryStats();
if ( rank == 0 && memory.N_new > memory.N_delete )
MemoryApp::print( std::cout );
#endif
}
/****************************************************************************
* Function to set an environemental variable *
****************************************************************************/
void Utilities::setenv( const std::string &name, const std::string &value )
{
Utilities_mutex.lock();
#if defined( USE_LINUX ) || defined( USE_MAC )
bool pass = false;
if ( !value.empty() )
pass = ::setenv( name.data(), value.data(), 1 ) == 0;
else
pass = ::unsetenv( name.data() ) == 0;
#elif defined( USE_WINDOWS )
bool pass = SetEnvironmentVariable( name.data(), value.data() ) != 0;
#else
#error Unknown OS
#endif
Utilities_mutex.unlock();
if ( !pass ) {
char msg[1024];
if ( !value.empty() )
sprintf(
msg, "Error setting enviornmental variable: %s=%s\n", name.data(), value.data() );
else
sprintf( msg, "Error clearing enviornmental variable: %s\n", name.data() );
ERROR( msg );
}
}
std::string Utilities::getenv( const std::string &name )
{
std::string var;
Utilities_mutex.lock();
auto tmp = std::getenv( name.data() );
if ( tmp )
var = std::string( tmp );
Utilities_mutex.unlock();
return var;
}
/****************************************************************************
* Factor a number into it's prime factors *
****************************************************************************/
std::vector<int> Utilities::factor(size_t number)
{
if ( number<=3 )
@ -54,9 +174,13 @@ std::vector<int> Utilities::factor(size_t number)
}
// Dummy function to prevent compiler from optimizing away variable
/****************************************************************************
* Dummy function to prevent compiler from optimizing away variable *
****************************************************************************/
void Utilities::nullUse( void* data )
{
NULL_USE(data);
}

31
common/Utilities.h
View File

@ -25,6 +25,37 @@ using StackTrace::Utilities::sleep_ms;
using StackTrace::Utilities::sleep_s;
/*!
* \brief Start MPI, error handlers
* \details This routine will peform the default startup sequence
* \param argc argc from main
* \param argv argv from main
*/
void startup( int argc, char **argv );
/*!
* \brief Stop MPI, error handlers
* \details This routine will peform the default shutdown sequence to match startup
*/
void shutdown();
/*!
* Get an environmental variable
* @param name The name of the environmental variable
* @return The value of the enviornmental variable
*/
std::string getenv( const std::string &name );
/*!
* Set an environmental variable
* @param name The name of the environmental variable
* @param value The value to set
*/
void setenv( const std::string &name, const std::string &value );
//! std::string version of sprintf
inline std::string stringf( const char *format, ... );

28
common/UtilityMacros.h
View File

@ -143,35 +143,43 @@
* Be sure to follow with ENABLE_WARNINGS
*/
// clang-format off
#ifdef DISABLE_WARNINGS
// Macros previously defined
#elif defined( USING_MSVC )
#ifndef DISABLE_WARNINGS
#if defined( USING_MSVC )
#define DISABLE_WARNINGS __pragma( warning( push, 0 ) )
#define ENABLE_WARNINGS __pragma( warning( pop ) )
#elif defined( USING_CLANG )
#define DISABLE_WARNINGS \
_Pragma( "clang diagnostic push" ) _Pragma( "clang diagnostic ignored \"-Wall\"" ) \
_Pragma( "clang diagnostic push" ) \
_Pragma( "clang diagnostic ignored \"-Wall\"" ) \
_Pragma( "clang diagnostic ignored \"-Wextra\"" ) \
_Pragma( "clang diagnostic ignored \"-Wunused-private-field\"" ) \
_Pragma( "clang diagnostic ignored \"-Wmismatched-new-delete\"" )
_Pragma( "clang diagnostic ignored \"-Wdeprecated-declarations\"" ) \
_Pragma( "clang diagnostic ignored \"-Winteger-overflow\"" )
#define ENABLE_WARNINGS _Pragma( "clang diagnostic pop" )
#elif defined( USING_GCC )
// Note: We cannot disable the -Wliteral-suffix message with this macro because the
// pragma command cannot suppress warnings from the C++ preprocessor. See gcc bug #53431.
#define DISABLE_WARNINGS \
_Pragma( "GCC diagnostic push" ) _Pragma( "GCC diagnostic ignored \"-Wall\"" ) \
_Pragma( "GCC diagnostic push" ) \
_Pragma( "GCC diagnostic ignored \"-Wpragmas\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wall\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wextra\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wpragmas\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wpedantic\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wunused-local-typedefs\"" ) \
_Pragma( "GCC diagnostic ignored \"-Woverloaded-virtual\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wunused-parameter\"" ) \
_Pragma( "GCC diagnostic ignored \"-Warray-bounds\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wdeprecated-declarations\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wvirtual-move-assign\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wunused-function\"" ) \
_Pragma( "GCC diagnostic ignored \"-Woverflow\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wunused-variable\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wignored-qualifiers\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wenum-compare\"" ) \
_Pragma( "GCC diagnostic ignored \"-Wterminate\"" )
#define ENABLE_WARNINGS _Pragma( "GCC diagnostic pop" )
#else
#define DISABLE_WARNINGS
#define ENABLE_WARNINGS
#endif
#endif
// clang-format on

5
cpu/BGK.cpp
View File

@ -1,5 +1,4 @@
extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
int n;
// conserved momemnts
double rho,ux,uy,uz,uu;
// non-conserved moments
@ -111,14 +110,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int
}
extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
int n;
// conserved momemnts
double rho,ux,uy,uz,uu;
// non-conserved moments
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18;
int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18;
int nread;
for (int n=start; n<finish; n++){
// q=0
@ -275,4 +272,4 @@ extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int star
rlx*0.02777777777777778*(rho - 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) - 0.08333333333*(Fy-Fz);
}
}
}

51
cpu/Color.cpp
View File

@ -920,21 +920,17 @@ extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A
double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC)
{
char id;
int idx,n,q,Cqx,Cqy,Cqz;
// int sendLoc;
double f0,f1,f2,f3,f4,f5,f6;
double na,nb,nab; // density values
double ux,uy,uz; // flow velocity
double nx,ny,nz,C; // color gradient components
double a1,a2,b1,b2;
double sp,delta;
double delta;
//double feq[6]; // equilibrium distributions
// Set of Discrete velocities for the D3Q19 Model
//int D3Q7[3][3]={{1,0,0},{0,1,0},{0,0,1}};
for (n=0; n<N; n++){
for (int n=0; n<N; n++){
id = ID[n];
if (id != 0 ){
@ -1224,25 +1220,20 @@ extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *P
extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N)
{
int n;
double Na,Nb;
//...................................................................
// Update Phi
for (n=0; n<N; n++){
for (int n=0; n<N; n++){
if (ID[n] > 0 ){
// Get the density value (Streaming already performed)
Na = Den[n];
Nb = Den[N+n];
double Na = Den[n];
double Nb = Den[N+n];
Phi[n] = (Na-Nb)/(Na+Nb);
}
}
//...................................................................
}
extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){
int n;
for (n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){
for (int n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){
Phi[n] = value;
}
}
@ -1255,7 +1246,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, do
double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){
int ijk,nn,n;
int ijk,nn;
double fq;
// conserved momemnts
double rho,jx,jy,jz;
@ -1838,7 +1829,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
double *Phi, double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){
int n,nn,ijk,nread;
int nn,ijk,nread;
int nr1,nr2,nr3,nr4,nr5,nr6;
int nr7,nr8,nr9,nr10;
int nr11,nr12,nr13,nr14;
@ -2492,7 +2483,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double *Aq, double *Bq,
double *Den, double *Phi, int start, int finish, int Np){
int idx,n,nread;
int idx, nread;
double fq,nA,nB;
for (int n=start; n<finish; n++){
@ -2578,11 +2569,11 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double
}
extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi,
int start, int finish, int Np){
int idx,n,nread;
double fq,nA,nB;
int start, int finish, int Np)
{
for (int n=start; n<finish; n++){
double fq,nA,nB;
// compute number density for component A
// q=0
fq = Aq[n];
@ -2646,27 +2637,25 @@ extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq,
Den[Np+n] = nB;
// save the phase indicator field
idx = Map[n];
int idx = Map[n];
Phi[idx] = (nA-nB)/(nA+nB);
}
}
extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *phi, double *ColorGrad, int start, int finish, int Np, int Nx, int Ny, int Nz){
int idx,n,N,i,j,k,nn;
// distributions
double f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double nx,ny,nz;
for (idx=0; idx<Np; idx++){
for (int idx=0; idx<Np; idx++){
// Get the 1D index based on regular data layout
n = Map[idx];
int n = Map[idx];
//.......Back out the 3D indices for node n..............
k = n/(Nx*Ny);
j = (n-Nx*Ny*k)/Nx;
i = n-Nx*Ny*k-Nx*j;
int k = n/(Nx*Ny);
int j = (n-Nx*Ny*k)/Nx;
int i = n-Nx*Ny*k-Nx*j;
//........................................................................
//........Get 1-D index for this thread....................
// n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
@ -2675,7 +2664,7 @@ extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *phi, double *ColorGrad,
//........................................................................
//.................Read Phase Indicator Values............................
//........................................................................
nn = n-1; // neighbor index (get convention)
int nn = n-1; // neighbor index (get convention)
if (i-1<0) nn += Nx; // periodic BC along the x-boundary
f1 = phi[nn]; // get neighbor for phi - 1
//........................................................................

654
cpu/D3Q19.cpp
View File

@ -270,8 +270,6 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
// odd distributions in disteven and even distributions in distodd.
int n,N;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double din = 0.f;
N = Nx*Ny*Nz;
@ -283,26 +281,26 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
// Read distributions from "opposite" memory convention
//........................................................................
//........................................................................
f1 = distodd[n];
f3 = distodd[N+n];
f5 = distodd[2*N+n];
f7 = distodd[3*N+n];
f9 = distodd[4*N+n];
f11 = distodd[5*N+n];
f13 = distodd[6*N+n];
f15 = distodd[7*N+n];
f17 = distodd[8*N+n];
double f1 = distodd[n];
double f3 = distodd[N+n];
//double f5 = distodd[2*N+n];
double f7 = distodd[3*N+n];
double f9 = distodd[4*N+n];
//double f11 = distodd[5*N+n];
double f13 = distodd[6*N+n];
//double f15 = distodd[7*N+n];
double f17 = distodd[8*N+n];
//........................................................................
f0 = disteven[n];
f2 = disteven[N+n];
f4 = disteven[2*N+n];
f6 = disteven[3*N+n];
f8 = disteven[4*N+n];
f10 = disteven[5*N+n];
f12 = disteven[6*N+n];
f14 = disteven[7*N+n];
f16 = disteven[8*N+n];
f18 = disteven[9*N+n];
double f0 = disteven[n];
double f2 = disteven[N+n];
double f4 = disteven[2*N+n];
double f6 = disteven[3*N+n];
double f8 = disteven[4*N+n];
double f10 = disteven[5*N+n];
double f12 = disteven[6*N+n];
//double f14 = disteven[7*N+n];
double f16 = disteven[8*N+n];
//double f18 = disteven[9*N+n];
//...................................................
// Determine the outlet flow velocity
@ -315,59 +313,58 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
}
extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *d_neighborList, int *list, double *dist, double flux,
double area, int count, int Np){
double area, int count, int Np)
{
int idx, n;
int nread;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double factor = 1.f/(area);
double sum = 0.f;
for (idx=0; idx<count; idx++){
n = list[idx];
f0 = dist[n];
double f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
double f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
double f3 = dist[nread];
nread = d_neighborList[n+6*Np];
f7 = dist[nread];
double f7 = dist[nread];
nread = d_neighborList[n+8*Np];
f9 = dist[nread];
double f9 = dist[nread];
nread = d_neighborList[n+12*Np];
f13 = dist[nread];
double f13 = dist[nread];
nread = d_neighborList[n+16*Np];
f17 = dist[nread];
double f17 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
double f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
double f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
double f6 = dist[nread];
nread = d_neighborList[n+7*Np];
f8 = dist[nread];
double f8 = dist[nread];
nread = d_neighborList[n+9*Np];
f10 = dist[nread];
double f10 = dist[nread];
nread = d_neighborList[n+11*Np];
f12 = dist[nread];
double f12 = dist[nread];
nread = d_neighborList[n+15*Np];
f16 = dist[nread];
double f16 = dist[nread];
sum += factor*(f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17));
}
@ -376,30 +373,29 @@ extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *d_neighborList, int *list, d
}
extern "C" double ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double flux, double area,
int count, int Np){
int count, int Np)
{
int idx, n;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double factor = 1.f/(area);
double sum = 0.f;
for (idx=0; idx<count; idx++){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
f7 = dist[8*Np+n];
f8 = dist[7*Np+n];
f9 = dist[10*Np+n];
f10 = dist[9*Np+n];
f12 = dist[11*Np+n];
f13 = dist[14*Np+n];
f16 = dist[15*Np+n];
f17 = dist[18*Np+n];
double f0 = dist[n];
double f1 = dist[2*Np+n];
double f2 = dist[1*Np+n];
double f3 = dist[4*Np+n];
double f4 = dist[3*Np+n];
double f6 = dist[5*Np+n];
double f7 = dist[8*Np+n];
double f8 = dist[7*Np+n];
double f9 = dist[10*Np+n];
double f10 = dist[9*Np+n];
double f12 = dist[11*Np+n];
double f13 = dist[14*Np+n];
double f16 = dist[15*Np+n];
double f17 = dist[18*Np+n];
sum += factor*(f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17));
}
return sum;
@ -412,8 +408,6 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub
// odd distributions in disteven and even distributions in distodd.
int n,N;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double dout = 0.f;
N = Nx*Ny*Nz;
@ -426,26 +420,26 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub
//........................................................................
// Read distributions from "opposite" memory convention
//........................................................................
f1 = distodd[n];
f3 = distodd[N+n];
f5 = distodd[2*N+n];
f7 = distodd[3*N+n];
f9 = distodd[4*N+n];
f11 = distodd[5*N+n];
f13 = distodd[6*N+n];
f15 = distodd[7*N+n];
f17 = distodd[8*N+n];
double f1 = distodd[n];
double f3 = distodd[N+n];
double f5 = distodd[2*N+n];
double f7 = distodd[3*N+n];
double f9 = distodd[4*N+n];
double f11 = distodd[5*N+n];
//double f13 = distodd[6*N+n];
double f15 = distodd[7*N+n];
//double f17 = distodd[8*N+n];
//........................................................................
f0 = disteven[n];
f2 = disteven[N+n];
f4 = disteven[2*N+n];
f6 = disteven[3*N+n];
f8 = disteven[4*N+n];
f10 = disteven[5*N+n];
f12 = disteven[6*N+n];
f14 = disteven[7*N+n];
f16 = disteven[8*N+n];
f18 = disteven[9*N+n];
double f0 = disteven[n];
double f2 = disteven[N+n];
double f4 = disteven[2*N+n];
//double f6 = disteven[3*N+n];
double f8 = disteven[4*N+n];
double f10 = disteven[5*N+n];
//double f12 = disteven[6*N+n];
double f14 = disteven[7*N+n];
//double f16 = disteven[8*N+n];
double f18 = disteven[9*N+n];
sum += (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f5+f11+f14+f15+f18));
@ -456,28 +450,25 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub
extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, double din, int count, int Np)
{
int idx, n;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double ux,uy,uz,Cyz,Cxz;
ux = uy = 0.0;
for (int idx=0; idx<count; idx++){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
f7 = dist[8*Np+n];
f8 = dist[7*Np+n];
f9 = dist[10*Np+n];
f10 = dist[9*Np+n];
f12 = dist[11*Np+n];
f13 = dist[14*Np+n];
f16 = dist[15*Np+n];
f17 = dist[18*Np+n];
int n = list[idx];
double f0 = dist[n];
double f1 = dist[2*Np+n];
double f2 = dist[1*Np+n];
double f3 = dist[4*Np+n];
double f4 = dist[3*Np+n];
double f6 = dist[5*Np+n];
double f7 = dist[8*Np+n];
double f8 = dist[7*Np+n];
double f9 = dist[10*Np+n];
double f10 = dist[9*Np+n];
double f12 = dist[11*Np+n];
double f13 = dist[14*Np+n];
double f16 = dist[15*Np+n];
double f17 = dist[18*Np+n];
//...................................................
// Determine the inlet flow velocity
//ux = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14);
@ -487,11 +478,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, doubl
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
f5 = f6 + 0.33333333333333338*uz;
f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
double f5 = f6 + 0.33333333333333338*uz;
double f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
double f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
double f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
double f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
dist[6*Np+n] = f5;
dist[12*Np+n] = f11;
@ -503,31 +494,28 @@ extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, doubl
extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, double dout, int count, int Np)
{
int idx, n;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double ux,uy,uz,Cyz,Cxz;
ux = uy = 0.0;
for (int idx=0; idx<count; idx++){
n = list[idx];
int n = list[idx];
//........................................................................
// Read distributions
//........................................................................
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f7 = dist[8*Np+n];
f8 = dist[7*Np+n];
f9 = dist[10*Np+n];
f10 = dist[9*Np+n];
f11 = dist[12*Np+n];
f14 = dist[13*Np+n];
f15 = dist[16*Np+n];
f18 = dist[17*Np+n];
double f0 = dist[n];
double f1 = dist[2*Np+n];
double f2 = dist[1*Np+n];
double f3 = dist[4*Np+n];
double f4 = dist[3*Np+n];
double f5 = dist[6*Np+n];
double f7 = dist[8*Np+n];
double f8 = dist[7*Np+n];
double f9 = dist[10*Np+n];
double f10 = dist[9*Np+n];
double f11 = dist[12*Np+n];
double f14 = dist[13*Np+n];
double f15 = dist[16*Np+n];
double f18 = dist[17*Np+n];
// Determine the outlet flow velocity
//ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
@ -537,11 +525,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, doubl
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
f6 = f5 - 0.33333333333333338*uz;
f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
double f6 = f5 - 0.33333333333333338*uz;
double f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
double f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
double f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
double f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
dist[5*Np+n] = f6;
dist[11*Np+n] = f12;
@ -554,57 +542,54 @@ extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, doubl
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np)
{
int idx, n;
int nread;
int nr5,nr11,nr14,nr15,nr18;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double ux,uy,uz,Cyz,Cxz;
ux = uy = 0.0;
for (int idx=0; idx<count; idx++){
n = list[idx];
f0 = dist[n];
int n = list[idx];
double f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
double f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
double f3 = dist[nread];
nread = d_neighborList[n+6*Np];
f7 = dist[nread];
double f7 = dist[nread];
nread = d_neighborList[n+8*Np];
f9 = dist[nread];
double f9 = dist[nread];
nread = d_neighborList[n+12*Np];
f13 = dist[nread];
double f13 = dist[nread];
nread = d_neighborList[n+16*Np];
f17 = dist[nread];
double f17 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
double f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
double f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
double f6 = dist[nread];
nread = d_neighborList[n+7*Np];
f8 = dist[nread];
double f8 = dist[nread];
nread = d_neighborList[n+9*Np];
f10 = dist[nread];
double f10 = dist[nread];
nread = d_neighborList[n+11*Np];
f12 = dist[nread];
double f12 = dist[nread];
nread = d_neighborList[n+15*Np];
f16 = dist[nread];
double f16 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
@ -622,11 +607,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list,
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
f5 = f6 + 0.33333333333333338*uz;
f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
double f5 = f6 + 0.33333333333333338*uz;
double f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
double f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
double f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
double f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
dist[nr5] = f5;
dist[nr11] = f11;
@ -638,60 +623,58 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list,
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list, double *dist, double dout, int count, int Np)
{
int idx,n,nread;
int nread;
int nr6,nr12,nr13,nr16,nr17;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double ux,uy,uz,Cyz,Cxz;
ux = uy = 0.0;
for (int idx=0; idx<count; idx++){
n = list[idx];
int n = list[idx];
//........................................................................
// Read distributions
//........................................................................
f0 = dist[n];
double f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
double f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
double f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
double f5 = dist[nread];
nread = d_neighborList[n+6*Np];
f7 = dist[nread];
double f7 = dist[nread];
nread = d_neighborList[n+8*Np];
f9 = dist[nread];
double f9 = dist[nread];
nread = d_neighborList[n+10*Np];
f11 = dist[nread];
double f11 = dist[nread];
nread = d_neighborList[n+14*Np];
f15 = dist[nread];
double f15 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
double f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
double f4 = dist[nread];
nread = d_neighborList[n+7*Np];
f8 = dist[nread];
double f8 = dist[nread];
nread = d_neighborList[n+9*Np];
f10 = dist[nread];
double f10 = dist[nread];
nread = d_neighborList[n+13*Np];
f14 = dist[nread];
double f14 = dist[nread];
nread = d_neighborList[n+17*Np];
f18 = dist[nread];
double f18 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
@ -708,11 +691,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list,
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
f6 = f5 - 0.33333333333333338*uz;
f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
double f6 = f5 - 0.33333333333333338*uz;
double f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
double f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
double f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
double f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
//........Store in "opposite" memory location..........
dist[nr6] = f6;
@ -724,145 +707,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list,
}
}
extern "C" void ScaLBL_D3Q19_Pressure_BC_z(int *list, double *dist, double din, int count, int Np)
{
int n;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double ux,uy,uz;
double Cxz,Cyz;
for (int idx=0; idx<count; idx++){
n = list[idx];
//........................................................................
// Read distributions from "opposite" memory convention
//........................................................................
//........................................................................
f0 = dist[n];
f1 = dist[Np+n];
f2 = dist[2*Np+n];
f3 = dist[3*Np+n];
f4 = dist[4*Np+n];
f6 = dist[6*Np+n];
f7 = dist[7*Np+n];
f8 = dist[8*Np+n];
f9 = dist[9*Np+n];
f10 = dist[10*Np+n];
f12 = dist[12*Np+n];
f13 = dist[13*Np+n];
f16 = dist[16*Np+n];
f17 = dist[17*Np+n];
//...................................................
//........Determine the inlet flow velocity.........
// uz = -1 + (f0+f3+f4+f1+f2+f7+f8+f10+f9
// + 2*(f5+f15+f18+f11+f14))/din;
//........Set the unknown distributions..............
// f6 = f5 - 0.3333333333333333*din*uz;
// f16 = f15 - 0.1666666666666667*din*uz;
// f17 = f16 - f3 + f4-f15+f18-f7+f8-f10+f9;
// f12= 0.5*(-din*uz+f5+f15+f18+f11+f14-f6-f16-
// f17+f1-f2-f14+f11+f7-f8-f10+f9);
// f13= -din*uz+f5+f15+f18+f11+f14-f6-f16-f17-f12;
// Determine the inlet flow velocity
ux = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14);
uy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18);
uz = din - (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17));
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
f5 = f6 + 0.33333333333333338*uz;
f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
//........Store in "opposite" memory location..........
dist[5*Np+n] = f5;
dist[11*Np+n] = f11;
dist[14*Np+n] = f14;
dist[15*Np+n] = f15;
dist[18*Np+n] = f18;
/*
printf("Site=%i\n",n);
printf("ux=%f, uy=%f, uz=%f\n",ux,uy,uz);
printf("Cxz=%f, Cyz=%f\n",Cxz,Cyz);
n = N;
*/
//...................................................
}
}
extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(int *list, double *dist, double dout, int count, int Np)
{
int n;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double ux,uy,uz;
double Cxz,Cyz;
for (int idx=0; idx<count; idx++){
n = list[idx];
//........................................................................
// Read distributions
//........................................................................
f0 = dist[n];
f1 = dist[Np+n];
f2 = dist[2*Np+n];
f3 = dist[3*Np+n];
f4 = dist[4*Np+n];
f5 = dist[5*Np+n];
f7 = dist[7*Np+n];
f8 = dist[8*Np+n];
f9 = dist[9*Np+n];
f10 = dist[10*Np+n];
f11 = dist[11*Np+n];
f14 = dist[14*Np+n];
f15 = dist[15*Np+n];
f18 = dist[18*Np+n];
//........Determine the outlet flow velocity.........
// uz = 1 - (f0+f3+f4+f1+f2+f7+f8+f10+f9+
// 2*(f6+f16+f17+f12+f13))/dout;
//...................................................
//........Set the Unknown Distributions..............
// f5 = f6 + 0.33333333333333338*dout*uz;
// f15 = f16 + 0.16666666666666678*dout*uz;
// f18 = f15+f3-f4-f16+f17+f7-f8+f10-f9;
// f11= 0.5*(dout*uz+f6+ f16+f17+f12+f13-f5
// -f15-f18-f1+f2-f13+f12-f7+f8+f10-f9);
// f14= dout*uz+f6+ f16+f17+f12+f13-f5-f15-f18-f11;
// Determine the outlet flow velocity
//ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
//uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
//uz = -1.0 + (f0+f4+f3+f2+f1+f8+f7+f9+f10 + 2*(f6+f16+f17+f12+f13))/dout;
// Determine the inlet flow velocity
ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
uz = -dout + (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f5+f11+f14+f15+f18));
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
f6 = f5 - 0.33333333333333338*uz;
f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
//........Store in "opposite" memory location..........
dist[6*Np+n] = f6;
dist[12*Np+n] = f12;
dist[13*Np+n] = f13;
dist[16*Np+n] = f16;
dist[17*Np+n] = f17;
//...................................................
}
}
extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz,
int Nx, int Ny, int Nz)
{
@ -1022,35 +866,30 @@ extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np)
extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *Pressure, int N)
{
int n;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
for (n=0; n<N; n++){
for (int n=0; n<N; n++){
//........................................................................
// Registers to store the distributions
//........................................................................
f0 = dist[n];
f2 = dist[2*N+n];
f4 = dist[4*N+n];
f6 = dist[6*N+n];
f8 = dist[8*N+n];
f10 = dist[10*N+n];
f12 = dist[12*N+n];
f14 = dist[14*N+n];
f16 = dist[16*N+n];
f18 = dist[18*N+n];
double f0 = dist[n];
double f2 = dist[2*N+n];
double f4 = dist[4*N+n];
double f6 = dist[6*N+n];
double f8 = dist[8*N+n];
double f10 = dist[10*N+n];
double f12 = dist[12*N+n];
double f14 = dist[14*N+n];
double f16 = dist[16*N+n];
double f18 = dist[18*N+n];
//........................................................................
f1 = dist[N+n];
f3 = dist[3*N+n];
f5 = dist[5*N+n];
f7 = dist[7*N+n];
f9 = dist[9*N+n];
f11 = dist[11*N+n];
f13 = dist[13*N+n];
f15 = dist[15*N+n];
f17 = dist[17*N+n];
double f1 = dist[N+n];
double f3 = dist[3*N+n];
double f5 = dist[5*N+n];
double f7 = dist[7*N+n];
double f9 = dist[9*N+n];
double f11 = dist[11*N+n];
double f13 = dist[13*N+n];
double f15 = dist[15*N+n];
double f17 = dist[17*N+n];
//.................Compute the velocity...................................
Pressure[n] = 0.3333333333333333*(f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+
f9+f12+f11+f14+f13+f16+f15+f18+f17);
@ -1058,31 +897,29 @@ extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *Pressure, int N)
}
extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
double Fy, double Fz){
int n;
double fq,fp;
double Fy, double Fz)
{
// conserved momemnts
double rho,jx,jy,jz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
const double mrt_V1=0.05263157894736842;
const double mrt_V2=0.012531328320802;
const double mrt_V3=0.04761904761904762;
const double mrt_V4=0.004594820384294068;
const double mrt_V5=0.01587301587301587;
const double mrt_V6=0.0555555555555555555555555;
const double mrt_V7=0.02777777777777778;
const double mrt_V8=0.08333333333333333;
const double mrt_V9=0.003341687552213868;
const double mrt_V10=0.003968253968253968;
const double mrt_V11=0.01388888888888889;
const double mrt_V12=0.04166666666666666;
constexpr double mrt_V1=0.05263157894736842;
constexpr double mrt_V2=0.012531328320802;
constexpr double mrt_V3=0.04761904761904762;
constexpr double mrt_V4=0.004594820384294068;
constexpr double mrt_V5=0.01587301587301587;
constexpr double mrt_V6=0.0555555555555555555555555;
constexpr double mrt_V7=0.02777777777777778;
constexpr double mrt_V8=0.08333333333333333;
constexpr double mrt_V9=0.003341687552213868;
constexpr double mrt_V10=0.003968253968253968;
constexpr double mrt_V11=0.01388888888888889;
constexpr double mrt_V12=0.04166666666666666;
for (int n=start; n<finish; n++){
// q=0
fq = dist[n];
double fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
@ -1489,31 +1326,30 @@ extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int
}
extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
double Fy, double Fz){
int n;
double fq,fp;
double Fy, double Fz)
{
// conserved momemnts
double rho,jx,jy,jz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
const double mrt_V1=0.05263157894736842;
const double mrt_V2=0.012531328320802;
const double mrt_V3=0.04761904761904762;
const double mrt_V4=0.004594820384294068;
const double mrt_V5=0.01587301587301587;
const double mrt_V6=0.0555555555555555555555555;
const double mrt_V7=0.02777777777777778;
const double mrt_V8=0.08333333333333333;
const double mrt_V9=0.003341687552213868;
const double mrt_V10=0.003968253968253968;
const double mrt_V11=0.01388888888888889;
const double mrt_V12=0.04166666666666666;
constexpr double mrt_V1=0.05263157894736842;
constexpr double mrt_V2=0.012531328320802;
constexpr double mrt_V3=0.04761904761904762;
constexpr double mrt_V4=0.004594820384294068;
constexpr double mrt_V5=0.01587301587301587;
constexpr double mrt_V6=0.0555555555555555555555555;
constexpr double mrt_V7=0.02777777777777778;
constexpr double mrt_V8=0.08333333333333333;
constexpr double mrt_V9=0.003341687552213868;
constexpr double mrt_V10=0.003968253968253968;
constexpr double mrt_V11=0.01388888888888889;
constexpr double mrt_V12=0.04166666666666666;
int nread;
for (int n=start; n<finish; n++){
// q=0
fq = dist[n];
double fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
@ -1964,11 +1800,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int star
}
}
extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *dist, int Np) {
int n;
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *dist, int Np)
{
for (int n=0; n<Np; n++){
@ -1977,27 +1810,26 @@ extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *dist, int Np) {
// (read from opposite array due to previous swap operation)
//........................................................................
// even
f2 = dist[10*Np+n];
f4 = dist[11*Np+n];
f6 = dist[12*Np+n];
f8 = dist[13*Np+n];
f10 = dist[14*Np+n];
f12 = dist[15*Np+n];
f14 = dist[16*Np+n];
f16 = dist[17*Np+n];
f18 = dist[18*Np+n];
double f2 = dist[10*Np+n];
double f4 = dist[11*Np+n];
double f6 = dist[12*Np+n];
double f8 = dist[13*Np+n];
double f10 = dist[14*Np+n];
double f12 = dist[15*Np+n];
double f14 = dist[16*Np+n];
double f16 = dist[17*Np+n];
double f18 = dist[18*Np+n];
f0 = dist[n];
// odd
f1 = dist[Np+n];
f3 = dist[2*Np+n];
f5 = dist[3*Np+n];
f7 = dist[4*Np+n];
f9 = dist[5*Np+n];
f11 = dist[6*Np+n];
f13 = dist[7*Np+n];
f15 = dist[8*Np+n];
f17 = dist[9*Np+n];
double f1 = dist[Np+n];
double f3 = dist[2*Np+n];
double f5 = dist[3*Np+n];
double f7 = dist[4*Np+n];
double f9 = dist[5*Np+n];
double f11 = dist[6*Np+n];
double f13 = dist[7*Np+n];
double f15 = dist[8*Np+n];
double f17 = dist[9*Np+n];
//........................................................................
// WRITE THE DISTRIBUTIONS
@ -2027,71 +1859,69 @@ extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *dist, int Np) {
}
}
extern "C" void ScaLBL_D3Q19_AAodd_Compact(char * ID, int *neighborList, double *dist, int Np) {
int n;
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
extern "C" void ScaLBL_D3Q19_AAodd_Compact(char * ID, int *neighborList, double *dist, int Np)
{
int nread;
for (int n=0; n<Np; n++){
//........Get 1-D index for this thread....................
f0 = dist[n];
//double f0 = dist[n];
nread = neighborList[n]; // + 0*Np
f2 = dist[nread];
double f2 = dist[nread];
nread = neighborList[n+2*Np];
f4 = dist[nread];
double f4 = dist[nread];
nread = neighborList[n+4*Np];
f6 = dist[nread];
double f6 = dist[nread];
nread = neighborList[n+6*Np];
f8 = dist[nread];
double f8 = dist[nread];
nread = neighborList[n+8*Np];
f10 = dist[nread];
double f10 = dist[nread];
nread = neighborList[n+10*Np];
f12 = dist[nread];
double f12 = dist[nread];
nread = neighborList[n+12*Np];
f14 = dist[nread];
double f14 = dist[nread];
nread = neighborList[n+14*Np];
f16 = dist[nread];
double f16 = dist[nread];
nread = neighborList[n+16*Np];
f18 = dist[nread];
double f18 = dist[nread];
nread = neighborList[n+Np];
f1 = dist[nread];
double f1 = dist[nread];
nread = neighborList[n+3*Np];
f3 = dist[nread];
double f3 = dist[nread];
nread = neighborList[n+5*Np];
f5 = dist[nread];
double f5 = dist[nread];
nread = neighborList[n+7*Np];
f7 = dist[nread];
double f7 = dist[nread];
nread = neighborList[n+9*Np];
f9 = dist[nread];
double f9 = dist[nread];
nread = neighborList[n+11*Np];
f11 = dist[nread];
double f11 = dist[nread];
nread = neighborList[n+13*Np];
f13 = dist[nread];
double f13 = dist[nread];
nread = neighborList[n+15*Np];
f15 = dist[nread];
double f15 = dist[nread];
nread = neighborList[n+17*Np];
f17 = dist[nread];
double f17 = dist[nread];
nread = neighborList[n];

33
cpu/dfh.cpp
View File

@ -23,11 +23,10 @@ extern "C" void ScaLBL_Gradient_Unpack(double weight, double Cqx, double Cqy, do
}
}
extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np){
int idx,n;
double phi,nA,nB;
for (idx=start; idx<finish; idx++){
extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np)
{
for (int idx=start; idx<finish; idx++){
double phi,nA,nB;
phi = Phi[idx];
if (phi > 0.f){
nA = 1.0; nB = 0.f;
@ -60,15 +59,13 @@ extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq
// LBM based on density functional hydrodynamics
extern "C" void ScaLBL_D3Q19_AAeven_DFH(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
double *Gradient, double *SolidForce, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
double Fx, double Fy, double Fz, int start, int finish, int Np){
int ijk,nn,n;
double Fx, double Fy, double Fz, int start, int finish, int Np)
{
double fq;
// conserved momemnts
double rho,jx,jy,jz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
double m3,m5,m7;
double nA,nB; // number density
double a1,b1,a2,b2,nAB,delta;
double C,nx,ny,nz; //color gradient magnitude and direction
@ -586,7 +583,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
double *Phi, double *Gradient, double *SolidForce, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
double Fx, double Fy, double Fz, int start, int finish, int Np){
int n,nn,ijk,nread;
int nread;
int nr1,nr2,nr3,nr4,nr5,nr6;
int nr7,nr8,nr9,nr10;
int nr11,nr12,nr13,nr14;
@ -596,7 +593,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
double rho,jx,jy,jz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
double m3,m5,m7;
double nA,nB; // number density
double a1,b1,a2,b2,nAB,delta;
double C,nx,ny,nz; //color gradient magnitude and direction
@ -1182,12 +1178,12 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
}
extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *neighborList, double *Aq, double *Bq,
double *Den, double *Phi, int start, int finish, int Np){
int idx,n,nread;
double fq,nA,nB;
double *Den, double *Phi, int start, int finish, int Np)
{
for (int n=start; n<finish; n++){
int nread;
double fq,nA,nB;
//..........Compute the number density for component A............
// q=0
@ -1270,11 +1266,10 @@ extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *neighborList, double *Aq, double *Bq,
}
extern "C" void ScaLBL_D3Q7_AAeven_DFH(double *Aq, double *Bq, double *Den, double *Phi,
int start, int finish, int Np){
int idx,n,nread;
double fq,nA,nB;
int start, int finish, int Np)
{
for (int n=start; n<finish; n++){
double fq,nA,nB;
// compute number density for component A
// q=0
fq = Aq[n];

540
cpu/exe/lb2_Color_mpi.cpp
View File

@ -4,7 +4,7 @@
#include "D3Q19.h"
#include "D3Q7.h"
#include "Color.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -36,15 +36,11 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -58,7 +54,6 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -115,31 +110,30 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
// **************************************************************
@ -169,7 +163,7 @@ int main(int argc, char **argv)
}
MPI_Barrier(comm);
comm.barrier();
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -451,7 +445,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@ -588,7 +582,7 @@ int main(int argc, char **argv)
}
}
}
MPI_Barrier(comm);
comm.barrier();
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -599,46 +593,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_x,1,rank_X,sendtag);
comm.Recv(&recvCount_X,1,rank_x,recvtag);
comm.Send(&sendCount_X,1,rank_x,sendtag);
comm.Recv(&recvCount_x,1,rank_X,recvtag);
comm.Send(&sendCount_y,1,rank_Y,sendtag);
comm.Recv(&recvCount_Y,1,rank_y,recvtag);
comm.Send(&sendCount_Y,1,rank_y,sendtag);
comm.Recv(&recvCount_y,1,rank_Y,recvtag);
comm.Send(&sendCount_z,1,rank_Z,sendtag);
comm.Recv(&recvCount_Z,1,rank_z,recvtag);
comm.Send(&sendCount_Z,1,rank_z,sendtag);
comm.Recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xy,1,rank_XY,sendtag);
comm.Recv(&recvCount_XY,1,rank_xy,recvtag);
comm.Send(&sendCount_XY,1,rank_xy,sendtag);
comm.Recv(&recvCount_xy,1,rank_XY,recvtag);
comm.Send(&sendCount_Xy,1,rank_xY,sendtag);
comm.Recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.Send(&sendCount_xY,1,rank_Xy,sendtag);
comm.Recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xz,1,rank_XZ,sendtag);
comm.Recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.Send(&sendCount_XZ,1,rank_xz,sendtag);
comm.Recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.Send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.Send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
comm.Send(&sendCount_yz,1,rank_YZ,sendtag);
comm.Recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.Send(&sendCount_YZ,1,rank_yz,sendtag);
comm.Recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.Send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.Send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
@ -669,48 +663,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -846,42 +840,24 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -914,7 +890,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
*/ //......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
MPI_Barrier(comm);
comm.barrier();
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -994,48 +970,49 @@ int main(int argc, char **argv)
PackValues(sendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
//...................................................................................
// Send / Recv all the phase indcator field values
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
//...................................................................................
UnpackValues(recvList_x, recvCount_x,recvbuf_x, Phi, N);
@ -1064,8 +1041,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
//.........................................
sendtag = recvtag = 5;
@ -1158,42 +1135,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//*************************************************************************
@ -1211,8 +1188,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1293,23 +1270,23 @@ int main(int argc, char **argv)
//...................................................................................
//...................................................................................
// Send all the D3Q7 distributions
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(recvbuf_x, 2*recvCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(sendbuf_X, 2*sendCount_X,rank_x,recvtag);
req1[1] = comm.Isend(recvbuf_X, 2*recvCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(sendbuf_x, 2*sendCount_x,rank_X,recvtag);
req1[2] = comm.Isend(recvbuf_y, 2*recvCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y, 2*sendCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(recvbuf_Y, 2*recvCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(sendbuf_y, 2*sendCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(recvbuf_z, 2*recvCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z, 2*sendCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(recvbuf_Z, 2*recvCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(sendbuf_z, 2*sendCount_z,rank_Z,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
comm.waitAll(6,req1);
comm.waitAll(6,req2);
//...................................................................................
//...................................................................................
UnpackDenD3Q7(sendList_x,sendCount_x,sendbuf_x,2,Den,N);
@ -1345,48 +1322,49 @@ int main(int argc, char **argv)
PackValues(sendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
//...................................................................................
// Send / Recv all the phase indcator field values
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
//...................................................................................
UnpackValues(recvList_x, recvCount_x,recvbuf_x, Phi, N);
@ -1409,14 +1387,14 @@ int main(int argc, char **argv)
UnpackValues(recvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
MPI_Barrier(comm);
comm.barrier();
// Iteration completed!
timestep++;
//...................................................................
}
//************************************************************************/
stoptime = MPI_Wtime();
stoptime = Utilities::MPI::time();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -1459,7 +1437,7 @@ int main(int argc, char **argv)
fwrite(ColorGrad,8,3*N,COLORGRAD);
fclose(COLORGRAD);
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

713
cpu/exe/lb2_Color_wia_mpi_bubble.cpp
View File

@ -10,7 +10,7 @@
#include "D3Q19.h"
#include "D3Q7.h"
#include "Color.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -62,7 +62,7 @@ inline void UnpackMeshData(int *list, int count, double *recvbuf, DoubleArray &V
}
}
//***************************************************************************************
inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
inline void CommunicateMeshHalo(DoubleArray &MeshData, const Utilities::MPI& Communicator,
double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy,
double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz,
@ -111,42 +111,24 @@ inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
PackMeshData(sendList_yZ, sendCount_yZ ,sendbuf_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData);
//......................................................................................
MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
recvbuf_X,recvCount_X,MPI_CHAR,rank_X,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
recvbuf_x,recvCount_x,MPI_CHAR,rank_x,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
recvbuf_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
recvbuf_y,recvCount_y,MPI_CHAR,rank_y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
recvbuf_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
recvbuf_z,recvCount_z,MPI_CHAR,rank_z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
recvbuf_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
recvbuf_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
recvbuf_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
recvbuf_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
recvbuf_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
recvbuf_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
recvbuf_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
recvbuf_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
recvbuf_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
recvbuf_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
recvbuf_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
recvbuf_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,Communicator,MPI_STATUS_IGNORE);
comm.sendrecv(sendbuf_x,sendCount_x,rank_x,sendtag,recvbuf_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendbuf_X,sendCount_X,rank_X,sendtag,recvbuf_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendbuf_y,sendCount_y,rank_y,sendtag,recvbuf_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendbuf_Y,sendCount_Y,rank_Y,sendtag,recvbuf_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendbuf_z,sendCount_z,rank_z,sendtag,recvbuf_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendbuf_Z,sendCount_Z,rank_Z,sendtag,recvbuf_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendbuf_xy,sendCount_xy,rank_xy,sendtag,recvbuf_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendbuf_XY,sendCount_XY,rank_XY,sendtag,recvbuf_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag,recvbuf_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendbuf_xY,sendCount_xY,rank_xY,sendtag,recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendbuf_xz,sendCount_xz,rank_xz,sendtag,recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag,recvbuf_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag,recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag,recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendbuf_yz,sendCount_yz,rank_yz,sendtag,recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag,recvbuf_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag,recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag,recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData);
@ -172,15 +154,11 @@ inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -194,7 +172,6 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -283,38 +260,39 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&xIntPos,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&xIntPos,1,0);
comm.bcast(&wp_saturation,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&Restart,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
// Computational domain
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
comm.bcast(&Nz,1,0);
// comm.bcast(&nBlocks,1,0);
// comm.bcast(&nthreads,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
// **************************************************************
double Ps = -(das-dbs)/(das+dbs);
@ -346,7 +324,7 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
MPI_Barrier(comm);
comm.barrier();
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -685,14 +663,14 @@ int main(int argc, char **argv)
//.......................................................................
if (rank == 0) printf("Reading the sphere packing \n");
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
MPI_Barrier(comm);
comm.barrier();
// Broadcast the sphere packing to all processes
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
//.......................................................................
// sprintf(LocalRankString,"%05d",rank);
@ -725,7 +703,7 @@ int main(int argc, char **argv)
}
}
sum_local = 1.0*sum;
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = comm.sumReduce( sum_local );
porosity = porosity*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -859,7 +837,7 @@ int main(int argc, char **argv)
}
}
}
MPI_Barrier(comm);
comm.barrier();
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -870,89 +848,48 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(&sendCount_x,1,rank_x,sendtag);
req2[0] = comm.Irecv(&recvCount_X,1,rank_X,recvtag);
req1[1] = comm.Isend(&sendCount_X,1,rank_X,sendtag);
req2[1] = comm.Irecv(&recvCount_x,1,rank_x,recvtag);
req1[2] = comm.Isend(&sendCount_y,1,rank_y,sendtag);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_Y,recvtag);
req1[3] = comm.Isend(&sendCount_Y,1,rank_Y,sendtag);
req2[3] = comm.Irecv(&recvCount_y,1,rank_y,recvtag);
req1[4] = comm.Isend(&sendCount_z,1,rank_z,sendtag);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_Z,recvtag);
req1[5] = comm.Isend(&sendCount_Z,1,rank_Z,sendtag);
req2[5] = comm.Irecv(&recvCount_z,1,rank_z,recvtag);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_xy,sendtag);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_XY,recvtag);
req1[7] = comm.Isend(&sendCount_XY,1,rank_XY,sendtag);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_xy,recvtag);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_Xy,sendtag);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_xY,recvtag);
req1[9] = comm.Isend(&sendCount_xY,1,rank_xY,sendtag);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_Xy,recvtag);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_xz,sendtag);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_XZ,recvtag);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_XZ,sendtag);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_xz,recvtag);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_Xz,sendtag);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_xZ,recvtag);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_xZ,sendtag);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_Xz,recvtag);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
*/ //**********************************************************************************
req1[14] = comm.Isend(&sendCount_yz,1,rank_yz,sendtag);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_YZ,recvtag);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_YZ,sendtag);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_yz,recvtag);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_Yz,sendtag);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_yZ,recvtag);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_yZ,sendtag);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_Yz,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@ -982,48 +919,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendList_Xz,endCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -1138,7 +1075,7 @@ int main(int argc, char **argv)
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
//......................................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
dvc_CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
dvc_CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
@ -1245,42 +1182,24 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
comm.sendrecv(sendID_x,sendCount_x,rank_x,sendtag,recvID_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_X,sendtag,recvID_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_y,sendtag,recvID_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_Y,sendtag,recvID_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_z,sendtag,recvID_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_Z,sendtag,recvID_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_xy,sendtag,recvID_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_XY,sendtag,recvID_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_Xy,sendtag,recvID_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_xY,sendtag,recvID_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_xz,sendtag,recvID_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_XZ,sendtag,recvID_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_Xz,sendtag,recvID_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_xZ,sendtag,recvID_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_yz,sendtag,recvID_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_YZ,sendtag,recvID_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_Yz,sendtag,recvID_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_yZ,sendtag,recvID_Yz,recvCount_Yz,rank_Yz,recvtag);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1348,7 +1267,7 @@ int main(int argc, char **argv)
recvMeshData_YZ = new double [recvCount_YZ];
recvMeshData_XZ = new double [recvCount_XZ];
if (rank==0) printf ("Devices are ready to communicate. \n");
MPI_Barrier(comm);
comm.barrier();
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1528,8 +1447,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
//.........................................
//...........................................................................
// MAIN VARIABLES INITIALIZED HERE
@ -1600,7 +1519,7 @@ int main(int argc, char **argv)
dvc_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
dvc_CopyToDevice(Den,cDen,2*N*sizeof(double));
dvc_Barrier();
MPI_Barrier(comm);
comm.barrier();
}
// Pack the buffers (zeros out the halo region)
dvc_PackDenD3Q7(dvcRecvList_x,recvCount_x,recvbuf_x,2,Den,N);
@ -1639,48 +1558,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
dvc_Barrier();
//...................................................................................
//...................................................................................
@ -1726,7 +1645,7 @@ int main(int argc, char **argv)
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double));
dvc_CopyToHost(Press.data,Pressure,N*sizeof(double));
dvc_CopyToHost(Vel,Velocity,3*N*sizeof(double));
MPI_Barrier(comm);
comm.barrier();
//...........................................................................
timestep=0;
@ -1822,42 +1741,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
//...................................................................................
//*************************************************************************
@ -1874,8 +1793,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
// Unpack the distributions on the device
@ -1958,23 +1877,23 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the D3Q7 distributions
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(recvbuf_x,2*recvCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(sendbuf_X,2*sendCount_X,rank_X,recvtag);
req1[1] = comm.Isend(recvbuf_X,2*recvCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(sendbuf_x,2*sendCount_x,rank_x,recvtag);
req1[2] = comm.Isend(recvbuf_y,2*recvCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y,2*sendCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(recvbuf_Y,2*recvCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(sendbuf_y,2*sendCount_y,rank_y,recvtag);
req1[4] = comm.Isend(recvbuf_z,2*recvCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z,2*sendCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(recvbuf_Z,2*recvCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(sendbuf_z,2*sendCount_z,rank_z,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
comm.waitAll(6,req1);
comm.waitAll(6,req2);
//...................................................................................
//...................................................................................
dvc_UnpackDenD3Q7(dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
@ -2013,48 +1932,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
dvc_Barrier();
//...................................................................................
//...................................................................................
@ -2084,7 +2003,7 @@ int main(int argc, char **argv)
dvc_UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
MPI_Barrier(comm);
comm.barrier();
// Iteration completed!
timestep++;
@ -2364,27 +2283,27 @@ int main(int argc, char **argv)
//...........................................................................
}
//...........................................................................
MPI_Barrier(comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
comm.barrier();
nwp_volume_global = comm.sumReduce( nwp_volume );
awn_global = comm.sumReduce( awn );
ans_global = comm.sumReduce( ans );
aws_global = comm.sumReduce( aws );
lwns_global = comm.sumReduce( lwns );
As_global = comm.sumReduce( As );
Jwn_global = comm.sumReduce( Jwn );
efawns_global = comm.sumReduce( efawns );
// Phase averages
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Barrier(comm);
vol_w_global = comm.sumReduce( vol_w );
vol_n_global = comm.sumReduce( vol_n );
paw_global = comm.sumReduce( paw );
pan_global = comm.sumReduce( pan );
vaw_global(0) = comm.sumReduce( vaw(0) );
van_global(0) = comm.sumReduce( van(0) );
vawn_global(0) = comm.sumReduce( vawn(0) );
Gwn_global(0) = comm.sumReduce( Gwn(0) );
Gns_global(0) = comm.sumReduce( Gns(0) );
Gws_global(0) = comm.sumReduce( Gws(0) );
comm.barrier();
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
// See McClure, Prins and Miller (2013)
@ -2451,8 +2370,8 @@ int main(int argc, char **argv)
}
//************************************************************************/
dvc_Barrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
@ -2489,7 +2408,7 @@ int main(int argc, char **argv)
*/ //************************************************************************/
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

348
gpu/exe/lb1_MRT_mpi.cpp
View File

@ -6,7 +6,7 @@
#include <iostream>
#include <fstream>
#include <string.h>
#include <mpi.h>
#include "common/MPI.h"
#include <stdlib.h>
using namespace std;
@ -64,15 +64,11 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -86,7 +82,6 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//**********************************
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@ -136,24 +131,23 @@ int main(int argc, char **argv)
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
double rlx_setA = 1.f/tau;
@ -176,7 +170,7 @@ int main(int argc, char **argv)
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
}
MPI_Barrier(comm);
comm.barrier();
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -457,7 +451,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@ -594,7 +588,7 @@ int main(int argc, char **argv)
}
}
}
MPI_Barrier(comm);
comm.barrier();
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -605,46 +599,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_x,1,rank_X,sendtag);
comm.recv(&recvCount_X,1,rank_x,recvtag);
comm.send(&sendCount_X,1,rank_x,sendtag);
comm.recv(&recvCount_x,1,rank_X,recvtag);
comm.send(&sendCount_y,1,rank_Y,sendtag);
comm.recv(&recvCount_Y,1,rank_y,recvtag);
comm.send(&sendCount_Y,1,rank_y,sendtag);
comm.recv(&recvCount_y,1,rank_Y,recvtag);
comm.send(&sendCount_z,1,rank_Z,sendtag);
comm.recv(&recvCount_Z,1,rank_z,recvtag);
comm.send(&sendCount_Z,1,rank_z,sendtag);
comm.recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xy,1,rank_XY,sendtag);
comm.recv(&recvCount_XY,1,rank_xy,recvtag);
comm.send(&sendCount_XY,1,rank_xy,sendtag);
comm.recv(&recvCount_xy,1,rank_XY,recvtag);
comm.send(&sendCount_Xy,1,rank_xY,sendtag);
comm.recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.send(&sendCount_xY,1,rank_Xy,sendtag);
comm.recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xz,1,rank_XZ,sendtag);
comm.recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.send(&sendCount_XZ,1,rank_xz,sendtag);
comm.recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
comm.send(&sendCount_yz,1,rank_YZ,sendtag);
comm.recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.send(&sendCount_YZ,1,rank_yz,sendtag);
comm.recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
@ -675,48 +669,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//......................................................................................
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
@ -915,42 +909,24 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -983,7 +959,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
//......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
MPI_Barrier(comm);
comm.barrier();
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1023,8 +999,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@ -1136,48 +1112,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1260,7 +1236,7 @@ int main(int argc, char **argv)
//*****************************************************************************
//*****************************************************************************
MPI_Barrier(comm);
comm.barrier();
// Iteration completed!
timestep++;
//...................................................................
@ -1269,8 +1245,8 @@ int main(int argc, char **argv)
// cudaThreadSynchronize();
dvc_Barrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -1304,7 +1280,7 @@ int main(int argc, char **argv)
// dvc_CopyToDevice(velocity, vel, 3*dist_mem_size, dvc_CopyToDeviceDeviceToHost);
//..............................................................................
// cudaThreadSynchronize();
// MPI_Barrier(comm);
// comm.barrier();
//............................................................
//....Write the z-velocity to test poiseuille flow............
// double vz,vz_avg;
@ -1333,7 +1309,7 @@ int main(int argc, char **argv)
// free (velocity); free(id);
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

352
gpu/exe/lb1_MRT_mpi.cu
View File

@ -1,8 +1,10 @@
#include "common/MPI.h"
#include <stdio.h>
#include <iostream>
#include <fstream>
#include <cuda.h>
#include <mpi.h>
inline void PackID(int *list, int count, char *sendbuf, char *ID){
// Fill in the phase ID values from neighboring processors
@ -553,15 +555,11 @@ void Write_Out(double *array, int Nx, int Ny, int Nz){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -575,7 +573,6 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//**********************************
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@ -625,24 +622,21 @@ int main(int argc, char **argv)
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&iterMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//.................................................
MPI_Barrier(comm);
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&iterMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
// **************************************************************
double rlx_setA = 1.f/tau;
@ -665,7 +659,7 @@ int main(int argc, char **argv)
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
}
MPI_Barrier(comm);
comm.barrier();
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -946,7 +940,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@ -1083,7 +1077,7 @@ int main(int argc, char **argv)
}
}
}
MPI_Barrier(comm);
comm.barrier();
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -1094,46 +1088,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_x,1,rank_X,sendtag);
comm.recv(&recvCount_X,1,rank_x,recvtag);
comm.send(&sendCount_X,1,rank_x,sendtag);
comm.recv(&recvCount_x,1,rank_X,recvtag);
comm.send(&sendCount_y,1,rank_Y,sendtag);
comm.recv(&recvCount_Y,1,rank_y,recvtag);
comm.send(&sendCount_Y,1,rank_y,sendtag);
comm.recv(&recvCount_y,1,rank_Y,recvtag);
comm.send(&sendCount_z,1,rank_Z,sendtag);
comm.recv(&recvCount_Z,1,rank_z,recvtag);
comm.send(&sendCount_Z,1,rank_z,sendtag);
comm.recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xy,1,rank_XY,sendtag);
comm.recv(&recvCount_XY,1,rank_xy,recvtag);
comm.send(&sendCount_XY,1,rank_xy,sendtag);
comm.recv(&recvCount_xy,1,rank_XY,recvtag);
comm.send(&sendCount_Xy,1,rank_xY,sendtag);
comm.recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.send(&sendCount_xY,1,rank_Xy,sendtag);
comm.recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xz,1,rank_XZ,sendtag);
comm.recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.send(&sendCount_XZ,1,rank_xz,sendtag);
comm.recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
comm.send(&sendCount_yz,1,rank_YZ,sendtag);
comm.recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.send(&sendCount_YZ,1,rank_yz,sendtag);
comm.recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
//**********************************************************************************
//recvCount_x = sendCount_x;
//recvCount_X = sendCount_X;
@ -1157,7 +1151,7 @@ int main(int argc, char **argv)
//......................................................................................
// Use MPI to fill in the appropriate values
// int tag = 5;
// MPI_Sendrecv(sendCount_x,1,MPI_INT,rank_x,tag,sendCount_X,1,MPI_INT,comm,req);
// Mcomm.sendrecv(sendCount_x,1,rank_x,tag,sendCount_X,1);
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@ -1187,48 +1181,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//......................................................................................
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
@ -1427,42 +1421,24 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1495,7 +1471,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
//......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
MPI_Barrier(comm);
comm.barrier();
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1535,8 +1511,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@ -1633,48 +1609,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1758,7 +1734,7 @@ int main(int argc, char **argv)
//*****************************************************************************
//*****************************************************************************
MPI_Barrier(comm);
comm.barrier();
// Iteration completed!
iter++;
//...................................................................
@ -1766,8 +1742,8 @@ int main(int argc, char **argv)
//************************************************************************/
cudaThreadSynchronize();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*iter)/cputime/1000000 << " MLUPS" << endl;
@ -1802,7 +1778,7 @@ int main(int argc, char **argv)
cudaMemcpy(velocity, vel, 3*dist_mem_size, cudaMemcpyDeviceToHost);
//..............................................................................
cudaThreadSynchronize();
MPI_Barrier(comm);
comm.barrier();
//............................................................
//....Write the z-velocity to test poiseuille flow............
double vz,vz_avg;
@ -1831,7 +1807,7 @@ int main(int argc, char **argv)
free (velocity); free(id);
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

65
gpu/exe/lb2_Color.cu
View File

@ -1,6 +1,4 @@
#ifdef useMPI
#include <mpi.h>
#endif
#include "common/MPI.h"
#include <stdio.h>
#include <stdlib.h>
@ -62,18 +60,10 @@ int main(int argc, char *argv[])
{
//********** Initialize MPI ****************
int numprocs,rank;
#ifdef useMPI
MPI_Status stat;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_size(comm,&numprocs);
MPI_Comm_rank(comm,&rank);
#else
MPI_Comm comm = MPI_COMM_WORLD;
numprocs = 1;
rank = 0;
#endif
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int numprocs = comm.getSize();
//******************************************
if (rank == 0){
@ -123,32 +113,31 @@ int main(int argc, char *argv[])
input >> tol; // error tolerance
//.............................................................
}
#ifdef useMPI
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
#endif
double rlxA = 1.f/tau;
double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA);
@ -243,11 +232,7 @@ int main(int argc, char *argv[])
if (k==4) k=Nz-5;
}
}
#ifdef useMPI //............................................................
MPI_Barrier(comm);
MPI_Bcast(&id[0],N,MPI_CHAR,0,comm);
MPI_Barrier(comm);
#endif
comm.bcast(&id[0],N,0);
if (rank == 0) printf("Domain set.\n");
//...........................................................................

541
gpu/exe/lb2_Color_mpi.cpp
View File

@ -2,7 +2,7 @@
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <mpi.h>
#include "common/MPI.h"
using namespace std;
@ -98,15 +98,11 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -120,7 +116,6 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -177,31 +172,30 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
// **************************************************************
@ -231,7 +225,7 @@ int main(int argc, char **argv)
}
MPI_Barrier(comm);
comm.barrier();
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -513,7 +507,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@ -650,7 +644,7 @@ int main(int argc, char **argv)
}
}
}
MPI_Barrier(comm);
comm.barrier();
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -661,46 +655,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_x,1,rank_X,sendtag);
comm.Recv(&recvCount_X,1,rank_x,recvtag);
comm.Send(&sendCount_X,1,rank_x,sendtag);
comm.Recv(&recvCount_x,1,rank_X,recvtag);
comm.Send(&sendCount_y,1,rank_Y,sendtag);
comm.Recv(&recvCount_Y,1,rank_y,recvtag);
comm.Send(&sendCount_Y,1,rank_y,sendtag);
comm.Recv(&recvCount_y,1,rank_Y,recvtag);
comm.Send(&sendCount_z,1,rank_Z,sendtag);
comm.Recv(&recvCount_Z,1,rank_z,recvtag);
comm.Send(&sendCount_Z,1,rank_z,sendtag);
comm.Recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xy,1,rank_XY,sendtag);
comm.Recv(&recvCount_XY,1,rank_xy,recvtag);
comm.Send(&sendCount_XY,1,rank_xy,sendtag);
comm.Recv(&recvCount_xy,1,rank_XY,recvtag);
comm.Send(&sendCount_Xy,1,rank_xY,sendtag);
comm.Recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.Send(&sendCount_xY,1,rank_Xy,sendtag);
comm.Recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xz,1,rank_XZ,sendtag);
comm.Recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.Send(&sendCount_XZ,1,rank_xz,sendtag);
comm.Recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.Send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.Send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
comm.Send(&sendCount_yz,1,rank_YZ,sendtag);
comm.Recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.Send(&sendCount_YZ,1,rank_yz,sendtag);
comm.Recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.Send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.Send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
@ -731,48 +725,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -993,42 +987,24 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1061,7 +1037,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
*/ //......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
MPI_Barrier(comm);
comm.barrier();
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1126,48 +1102,49 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@ -1207,8 +1184,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@ -1309,42 +1286,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//*************************************************************************
@ -1363,8 +1340,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1446,23 +1423,23 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the D3Q7 distributions
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(recvbuf_x, 2*recvCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(sendbuf_X, 2*sendCount_X,rank_x,recvtag);
req1[1] = comm.Isend(recvbuf_X, 2*recvCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(sendbuf_x, 2*sendCount_x,rank_X,recvtag);
req1[2] = comm.Isend(recvbuf_y, 2*recvCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y, 2*sendCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(recvbuf_Y, 2*recvCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(sendbuf_y, 2*sendCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(recvbuf_z, 2*recvCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z, 2*sendCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(recvbuf_Z, 2*recvCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(sendbuf_z, 2*sendCount_z,rank_Z,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
comm.waitAll(6,req1);
comm.waitAll(6,req2);
//...................................................................................
//...................................................................................
dvc_UnpackDenD3Q7(faceGrid,packThreads,dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
@ -1507,48 +1484,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@ -1577,7 +1554,7 @@ int main(int argc, char **argv)
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
MPI_Barrier(comm);
comm.barrier();
// Iteration completed!
timestep++;
@ -1587,8 +1564,8 @@ int main(int argc, char **argv)
// cudaThreadSynchronize();
dvc_Barrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -1616,7 +1593,7 @@ int main(int argc, char **argv)
//************************************************************************/
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

621
gpu/exe/lb2_Color_pBC_wia_mpi.cpp
View File

@ -2,7 +2,7 @@
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <mpi.h>
#include "common/MPI.h"
#include "pmmc.h"
#include "Domain.h"
@ -101,15 +101,11 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -123,7 +119,6 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -203,35 +198,35 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
// Computational domain
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
// **************************************************************
double Ps = -(das-dbs)/(das+dbs);
@ -263,7 +258,7 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
MPI_Barrier(comm);
comm.barrier();
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -561,14 +556,14 @@ int main(int argc, char **argv)
//.......................................................................
if (rank == 0) printf("Reading the sphere packing \n");
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
MPI_Barrier(comm);
comm.barrier();
// Broadcast the sphere packing to all processes
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
//.......................................................................
// sprintf(LocalRankString,"%05d",rank);
@ -718,7 +713,7 @@ int main(int argc, char **argv)
}
}
}
MPI_Barrier(comm);
comm.barrier();
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -729,89 +724,49 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(&sendCount_x,1,rank_X,sendtag);
req2[0] = comm.Irecv(&recvCount_X,1,rank_x,recvtag);
req1[1] = comm.Isend(&sendCount_X,1,rank_x,sendtag);
req2[1] = comm.Irecv(&recvCount_x,1,rank_X,recvtag);
req1[2] = comm.Isend(&sendCount_y,1,rank_Y,sendtag);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_y,recvtag);
req1[3] = comm.Isend(&sendCount_Y,1,rank_y,sendtag);
req2[3] = comm.Irecv(&recvCount_y,1,rank_Y,recvtag);
req1[4] = comm.Isend(&sendCount_z,1,rank_Z,sendtag);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_z,recvtag);
req1[5] = comm.Isend(&sendCount_Z,1,rank_z,sendtag);
req2[5] = comm.Irecv(&recvCount_z,1,rank_Z,recvtag);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_XY,sendtag);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_xy,recvtag);
req1[7] = comm.Isend(&sendCount_XY,1,rank_xy,sendtag);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_XY,recvtag);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_xY,sendtag);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_Xy,recvtag);
req1[9] = comm.Isend(&sendCount_xY,1,rank_Xy,sendtag);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_XZ,sendtag);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_xz,recvtag);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_xz,sendtag);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_XZ,recvtag);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_xZ,sendtag);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_Xz,recvtag);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_Xz,sendtag);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
*/ //**********************************************************************************
req1[14] = comm.Isend(&sendCount_yz,1,rank_YZ,sendtag);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_yz,recvtag);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_yz,sendtag);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_YZ,recvtag);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_yZ,sendtag);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_Yz,recvtag);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_Yz,sendtag);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@ -841,48 +796,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -1103,42 +1058,24 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1171,7 +1108,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
*/ //......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
MPI_Barrier(comm);
comm.barrier();
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1322,48 +1259,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@ -1403,8 +1340,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
//.........................................
sendtag = recvtag = 5;
@ -1494,42 +1431,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//*************************************************************************
@ -1547,8 +1484,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1630,23 +1567,23 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the D3Q7 distributions
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
req1[0] = comm.Isend(recvbuf_x,2*recvCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(sendbuf_X,2*sendCount_X,rank_x,recvtag);
req1[1] = comm.Isend(recvbuf_X,2*recvCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(sendbuf_x,2*sendCount_x,rank_X,recvtag);
req1[2] = comm.Isend(recvbuf_y,2*recvCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y,2*sendCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(recvbuf_Y,2*recvCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(sendbuf_y,2*sendCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(recvbuf_z,2*recvCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z,2*sendCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(recvbuf_Z,2*recvCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(sendbuf_z,2*sendCount_z,rank_Z,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
comm.waitAll(6,req1);
comm.waitAll(6,req2);
//...................................................................................
//...................................................................................
dvc_UnpackDenD3Q7(faceGrid,packThreads,dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
@ -1685,48 +1622,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@ -1755,7 +1692,7 @@ int main(int argc, char **argv)
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
MPI_Barrier(comm);
comm.barrier();
// Iteration completed!
timestep++;
//...................................................................
@ -1766,7 +1703,7 @@ int main(int argc, char **argv)
//...........................................................................
dvc_Barrier();
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double));
MPI_Barrier(comm);
comm.barrier();
//...........................................................................
// Compute areas using porous medium marching cubes algorithm
// McClure, Adalsteinsson, et al. (2007)
@ -1935,15 +1872,15 @@ int main(int argc, char **argv)
//*******************************************************************
}
//...........................................................................
MPI_Barrier(comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
comm.barrier();
nwp_volume_global = comm.sumReduce( nwp_volume );
awn_global = comm.sumReduce( awn );
ans_global = comm.sumReduce( ans );
aws_global = comm.sumReduce( aws );
lwns_global = comm.sumReduce( lwns );
As_global = comm.sumReduce( As );
MPI_Barrier(comm);
comm.barrier();
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
// See McClure, Prins and Miller (2013)
@ -1972,8 +1909,8 @@ int main(int argc, char **argv)
}
//************************************************************************/
dvc_Barrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
if (rank==0) printf("-------------------------------------------------------------------\n");
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
@ -2009,7 +1946,7 @@ int main(int argc, char **argv)
*/ //************************************************************************/
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

117
models/ColorModel.cpp
View File

@ -4,10 +4,12 @@ color lattice boltzmann model
#include "models/ColorModel.h"
#include "analysis/distance.h"
#include "analysis/morphology.h"
#include "common/Communication.h"
#include "common/ReadMicroCT.h"
#include <stdlib.h>
#include <time.h>
ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM):
ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0),
Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),
Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
@ -118,7 +120,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){
inletB=0.f;
outletA=0.f;
outletB=1.f;
if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details)
//if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details)
BoundaryCondition = 0;
if (domain_db->keyExists( "BC" )){
@ -165,9 +167,9 @@ void ScaLBL_ColorModel::SetDomain(){
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1; // initialize this way
//Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
Averages = std::shared_ptr<SubPhase> ( new SubPhase(Dm) ); // TwoPhase analysis object
MPI_Barrier(comm);
comm.barrier();
Dm->CommInit();
MPI_Barrier(comm);
comm.barrier();
// Read domain parameters
rank = Dm->rank();
nprocx = Dm->nprocx();
@ -184,11 +186,23 @@ void ScaLBL_ColorModel::ReadInput(){
if (color_db->keyExists( "image_sequence" )){
auto ImageList = color_db->getVector<std::string>( "image_sequence");
int IMAGE_INDEX = color_db->getWithDefault<int>( "image_index", 0 );
int IMAGE_COUNT = ImageList.size();
std::string first_image = ImageList[IMAGE_INDEX];
Mask->Decomp(first_image);
IMAGE_INDEX++;
}
else if (domain_db->keyExists( "GridFile" )){
// Read the local domain data
auto input_id = readMicroCT( *domain_db, comm );
// Fill the halo (assuming GCW of 1)
array<int,3> size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) };
ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz };
ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 );
fillHalo<signed char> fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 );
Array<signed char> id_view;
id_view.viewRaw( size1, Mask->id );
fill.copy( input_id, id_view );
fill.fill( id_view );
}
else if (domain_db->keyExists( "Filename" )){
auto Filename = domain_db->getScalar<std::string>( "Filename" );
Mask->Decomp(Filename);
@ -201,7 +215,6 @@ void ScaLBL_ColorModel::ReadInput(){
// Generate the signed distance map
// Initialize the domain and communication
Array<char> id_solid(Nx,Ny,Nz);
int count = 0;
// Solve for the position of the solid phase
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
@ -218,7 +231,6 @@ void ScaLBL_ColorModel::ReadInput(){
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
for (int i=0;i<Nx;i++){
int n=k*Nx*Ny+j*Nx+i;
// Initialize distance to +/- 1
Averages->SDs(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0;
}
@ -251,7 +263,7 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase)
double label_count_global[NLABELS];
// Assign the labels
for (int idx=0; idx<NLABELS; idx++) label_count[idx]=0;
for (size_t idx=0; idx<NLABELS; idx++) label_count[idx]=0;
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
@ -279,7 +291,8 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase)
// Set Dm to match Mask
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i];
for (int idx=0; idx<NLABELS; idx++) label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]);
for (size_t idx=0; idx<NLABELS; idx++)
label_count_global[idx] = Dm->Comm.sumReduce( label_count[idx] );
if (rank==0){
printf("Component labels: %lu \n",NLABELS);
@ -320,7 +333,7 @@ void ScaLBL_ColorModel::Create(){
Map.resize(Nx,Ny,Nz); Map.fill(-2);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
MPI_Barrier(comm);
comm.barrier();
//...........................................................................
// MAIN VARIABLES ALLOCATED HERE
@ -358,16 +371,16 @@ void ScaLBL_ColorModel::Create(){
}
// check that TmpMap is valid
for (int idx=0; idx<ScaLBL_Comm->LastExterior(); idx++){
int n = TmpMap[idx];
auto n = TmpMap[idx];
if (n > Nx*Ny*Nz){
printf("Bad value! idx=%i \n");
printf("Bad value! idx=%i \n", n);
TmpMap[idx] = Nx*Ny*Nz-1;
}
}
for (int idx=ScaLBL_Comm->FirstInterior(); idx<ScaLBL_Comm->LastInterior(); idx++){
int n = TmpMap[idx];
if (n > Nx*Ny*Nz){
printf("Bad value! idx=%i \n");
auto n = TmpMap[idx];
if ( n > Nx*Ny*Nz ){
printf("Bad value! idx=%i \n",n);
TmpMap[idx] = Nx*Ny*Nz-1;
}
}
@ -452,7 +465,7 @@ void ScaLBL_ColorModel::Initialize(){
ScaLBL_CopyToDevice(Phi,cPhi,N*sizeof(double));
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
}
if (rank==0) printf ("Initializing phase field \n");
@ -538,8 +551,9 @@ void ScaLBL_ColorModel::Run(){
}
if (color_db->keyExists( "residual_endpoint_threshold" )){
RESIDUAL_ENDPOINT_THRESHOLD = color_db->getScalar<double>( "residual_endpoint_threshold" );
RESIDUAL_ENDPOINT_THRESHOLD = color_db->getScalar<double>( "residual_endpoint_threshold" );
}
NULL_USE( RESIDUAL_ENDPOINT_THRESHOLD );
if (color_db->keyExists( "noise_threshold" )){
NOISE_THRESHOLD = color_db->getScalar<double>( "noise_threshold" );
USE_BUMP_RATE = true;
@ -637,8 +651,8 @@ void ScaLBL_ColorModel::Run(){
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
@ -686,7 +700,8 @@ void ScaLBL_ColorModel::Run(){
}
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier();
comm.barrier();
// *************EVEN TIMESTEP*************
timestep++;
@ -721,10 +736,10 @@ void ScaLBL_ColorModel::Run(){
}
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier();
comm.barrier();
//************************************************************************
MPI_Barrier(comm);
PROFILE_STOP("Update");
if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition > 0){
@ -859,7 +874,7 @@ void ScaLBL_ColorModel::Run(){
WriteHeader=true;
kr_log_file = fopen("relperm.csv","a");
if (WriteHeader)
fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,pAB,viscous_pressure_drop,Ca,Mobility);
fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n");
fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility);
fclose(kr_log_file);
@ -923,7 +938,7 @@ void ScaLBL_ColorModel::Run(){
delta_volume = volA*Dm->Volume - initial_volume;
CURRENT_MORPH_TIMESTEPS += analysis_interval;
double massChange = SeedPhaseField(seed_water);
if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", seed_water, delta_volume, delta_volume_target);
if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target);
}
else if (USE_MORPHOPEN_OIL){
delta_volume = volA*Dm->Volume - initial_volume;
@ -965,7 +980,7 @@ void ScaLBL_ColorModel::Run(){
//morph_delta *= (-1.0);
REVERSE_FLOW_DIRECTION = false;
}
MPI_Barrier(comm);
comm.barrier();
}
morph_timesteps += analysis_interval;
}
@ -975,8 +990,8 @@ void ScaLBL_ColorModel::Run(){
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
@ -995,7 +1010,6 @@ void ScaLBL_ColorModel::Run(){
double ScaLBL_ColorModel::ImageInit(std::string Filename){
bool suppress = false;
if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str());
Mask->Decomp(Filename);
for (int i=0; i<Nx*Ny*Nz; i++) id[i] = Mask->id[i]; // save what was read
@ -1021,17 +1035,17 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){
}
}
Count=sumReduce( Dm->Comm, Count);
PoreCount=sumReduce( Dm->Comm, PoreCount);
Count = Dm->Comm.sumReduce( Count );
PoreCount = Dm->Comm.sumReduce( PoreCount );
if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount);
ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double));
MPI_Barrier(comm);
comm.barrier();
ScaLBL_D3Q19_Init(fq, Np);
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
MPI_Barrier(comm);
comm.barrier();
ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double));
@ -1063,12 +1077,11 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
BlobIDstruct new_index;
double vF=0.0; double vS=0.0;
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm);
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
int count_oil=0;
int count_connected=0;
int count_porespace=0;
int count_water=0;
long long count_connected=0;
long long count_porespace=0;
long long count_water=0;
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){
for (int i=1; i<nx-1; i++){
@ -1086,9 +1099,9 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
}
}
}
count_connected=sumReduce( Dm->Comm, count_connected);
count_porespace=sumReduce( Dm->Comm, count_porespace);
count_water=sumReduce( Dm->Comm, count_water);
count_connected = Dm->Comm.sumReduce( count_connected);
count_porespace = Dm->Comm.sumReduce( count_porespace);
count_water = Dm->Comm.sumReduce( count_water);
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
@ -1160,7 +1173,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
}
}
}
count_morphopen=sumReduce( Dm->Comm, count_morphopen);
count_morphopen = Dm->Comm.sumReduce( count_morphopen);
volume_change = double(count_morphopen - count_connected);
if (rank==0) printf(" opening of connected oil %f \n",volume_change/count_connected);
@ -1266,8 +1279,8 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){
mass_loss += random_value*seed_water_in_oil;
}
count= sumReduce( Dm->Comm, count);
mass_loss= sumReduce( Dm->Comm, mass_loss);
count = Dm->Comm.sumReduce( count );
mass_loss = Dm->Comm.sumReduce( mass_loss );
if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count);
// Need to initialize Aq, Bq, Den, Phi directly
@ -1296,8 +1309,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
// 1. Copy phase field to CPU
ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double));
double count,count_global,volume_initial,volume_final,volume_connected;
count = 0.f;
double count = 0.f;
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
@ -1305,7 +1317,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
}
}
}
volume_initial = sumReduce( Dm->Comm, count);
double volume_initial = Dm->Comm.sumReduce( count);
/*
sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank);
FILE *INPUT = fopen(LocalRankFilename,"wb");
@ -1315,7 +1327,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
// 2. Identify connected components of phase field -> phase_label
BlobIDstruct new_index;
ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm);
MPI_Barrier(comm);
comm.barrier();
// only operate on component "0"
count = 0.0;
@ -1337,16 +1349,16 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
}
}
}
volume_connected = sumReduce( Dm->Comm, count);
second_biggest = sumReduce( Dm->Comm, second_biggest);
double volume_connected = Dm->Comm.sumReduce( count );
second_biggest = Dm->Comm.sumReduce( second_biggest );
int reach_x, reach_y, reach_z;
/*int reach_x, reach_y, reach_z;
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
for (int i=0; i<Nx; i++){
}
}
}
}*/
// 3. Generate a distance map to the largest object -> phase_distance
CalcDist(phase_distance,phase_id,*Dm);
@ -1402,7 +1414,6 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
for (int i=0; i<Nx; i++){
int n = k*Nx*Ny + j*Nx + i;
double d = phase_distance(i,j,k);
if (Averages->SDs(i,j,k) > 0.f){
if (d < 3.f){
@ -1426,7 +1437,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
}
}
}
volume_final= sumReduce( Dm->Comm, count);
double volume_final = Dm->Comm.sumReduce( count );
delta_volume = (volume_final-volume_initial);
if (rank == 0) printf("MorphInit: change fluid volume fraction by %f \n", delta_volume/volume_initial);

6
models/ColorModel.h
View File

@ -12,13 +12,13 @@ Implementation of color lattice boltzmann model
#include "common/Communication.h"
#include "analysis/TwoPhase.h"
#include "analysis/runAnalysis.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
class ScaLBL_ColorModel{
public:
ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM);
ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM);
~ScaLBL_ColorModel();
// functions in they should be run
@ -68,7 +68,7 @@ public:
double *Pressure;
private:
MPI_Comm comm;
Utilities::MPI comm;
int dist_mem_size;
int neighborSize;

40
models/DFHModel.cpp
View File

@ -3,7 +3,7 @@ color lattice boltzmann model
*/
#include "models/DFHModel.h"
ScaLBL_DFHModel::ScaLBL_DFHModel(int RANK, int NP, MPI_Comm COMM):
ScaLBL_DFHModel::ScaLBL_DFHModel(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0),
Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),
Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
@ -100,21 +100,20 @@ void ScaLBL_DFHModel::ReadParams(string filename){
}
void ScaLBL_DFHModel::SetDomain(){
Dm = std::shared_ptr<Domain>(new Domain(domain_db,comm)); // full domain for analysis
Mask = std::shared_ptr<Domain>(new Domain(domain_db,comm)); // mask domain removes immobile phases
Dm = std::make_shared<Domain>(domain_db,comm); // full domain for analysis
Mask = std::make_shared<Domain>(domain_db,comm); // mask domain removes immobile phases
Nx+=2; Ny+=2; Nz += 2;
N = Nx*Ny*Nz;
id = new char [N];
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1; // initialize this way
Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
MPI_Barrier(comm);
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1; // initialize this way
Averages = std::make_shared<TwoPhase>( Dm ); // TwoPhase analysis object
comm.barrier();
Dm->CommInit();
MPI_Barrier(comm);
comm.barrier();
rank = Dm->rank();
}
void ScaLBL_DFHModel::ReadInput(){
size_t readID;
//.......................................................................
if (rank == 0) printf("Read input media... \n");
//.......................................................................
@ -132,7 +131,7 @@ void ScaLBL_DFHModel::ReadInput(){
sprintf(LocalRankString,"%05d",rank);
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N);
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
}
@ -207,7 +206,7 @@ void ScaLBL_DFHModel::Create(){
Map.resize(Nx,Ny,Nz); Map.fill(-2);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
MPI_Barrier(comm);
comm.barrier();
//...........................................................................
// MAIN VARIABLES ALLOCATED HERE
@ -425,7 +424,7 @@ void ScaLBL_DFHModel::Initialize(){
}
}
}
MPI_Allreduce(&count_wet,&count_wet_global,1,MPI_DOUBLE,MPI_SUM,comm);
count_wet_global = comm.sumReduce( count_wet );
if (rank==0) printf("Wetting phase volume fraction =%f \n",count_wet_global/double(Nx*Ny*Nz*nprocs));
// initialize phi based on PhaseLabel (include solid component labels)
ScaLBL_CopyToDevice(Phi, PhaseLabel, Np*sizeof(double));
@ -447,7 +446,7 @@ void ScaLBL_DFHModel::Initialize(){
timestep=0;
}
}
MPI_Bcast(&timestep,1,MPI_INT,0,comm);
comm.bcast(&timestep,1,0);
// Read in the restart file to CPU buffers
double *cPhi = new double[Np];
double *cDist = new double[19*Np];
@ -469,7 +468,7 @@ void ScaLBL_DFHModel::Initialize(){
ScaLBL_DeviceBarrier();
delete [] cPhi;
delete [] cDist;
MPI_Barrier(comm);
comm.barrier();
}
if (rank==0) printf ("Initializing phase field \n");
@ -487,8 +486,8 @@ void ScaLBL_DFHModel::Run(){
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
@ -533,7 +532,8 @@ void ScaLBL_DFHModel::Run(){
}
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier();
comm.barrier();
// *************EVEN TIMESTEP*************
timestep++;
@ -569,9 +569,9 @@ void ScaLBL_DFHModel::Run(){
}
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier();
comm.barrier();
//************************************************************************
MPI_Barrier(comm);
PROFILE_STOP("Update");
// Run the analysis
@ -582,8 +582,8 @@ void ScaLBL_DFHModel::Run(){
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

6
models/DFHModel.h
View File

@ -12,13 +12,13 @@ Implementation of color lattice boltzmann model
#include "common/Communication.h"
#include "analysis/TwoPhase.h"
#include "analysis/runAnalysis.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
class ScaLBL_DFHModel{
public:
ScaLBL_DFHModel(int RANK, int NP, MPI_Comm COMM);
ScaLBL_DFHModel(int RANK, int NP, const Utilities::MPI& COMM);
~ScaLBL_DFHModel();
// functions in they should be run
@ -66,7 +66,7 @@ public:
double *Pressure;
private:
MPI_Comm comm;
Utilities::MPI comm;
int dist_mem_size;
int neighborSize;

84
models/MRTModel.cpp
View File

@ -4,7 +4,7 @@
#include "models/MRTModel.h"
#include "analysis/distance.h"
ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM):
ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),
Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0),
Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
@ -82,9 +82,9 @@ void ScaLBL_MRTModel::SetDomain(){
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1; // initialize this way
//Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
MPI_Barrier(comm);
comm.barrier();
Dm->CommInit();
MPI_Barrier(comm);
comm.barrier();
rank = Dm->rank();
nprocx = Dm->nprocx();
@ -93,20 +93,22 @@ void ScaLBL_MRTModel::SetDomain(){
}
void ScaLBL_MRTModel::ReadInput(){
int rank=Dm->rank();
size_t readID;
//.......................................................................
//.......................................................................
Mask->ReadIDs();
sprintf(LocalRankString,"%05d",Dm->rank());
sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
if (domain_db->keyExists( "Filename" )){
auto Filename = domain_db->getScalar<std::string>( "Filename" );
Mask->Decomp(Filename);
}
else{
Mask->ReadIDs();
}
// Generate the signed distance map
// Initialize the domain and communication
Array<char> id_solid(Nx,Ny,Nz);
int count = 0;
// Solve for the position of the solid phase
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
@ -122,7 +124,6 @@ void ScaLBL_MRTModel::ReadInput(){
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
for (int i=0;i<Nx;i++){
int n=k*Nx*Ny+j*Nx+i;
// Initialize distance to +/- 1
Distance(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0;
}
@ -155,7 +156,7 @@ void ScaLBL_MRTModel::Create(){
Map.resize(Nx,Ny,Nz); Map.fill(-2);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
MPI_Barrier(comm);
comm.barrier();
//...........................................................................
// MAIN VARIABLES ALLOCATED HERE
//...........................................................................
@ -174,7 +175,7 @@ void ScaLBL_MRTModel::Create(){
if (rank==0) printf ("Setting up device map and neighbor list \n");
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
MPI_Barrier(comm);
comm.barrier();
}
@ -191,18 +192,27 @@ void ScaLBL_MRTModel::Run(){
double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
Minkowski Morphology(Mask);
int SIZE=Np*sizeof(double);
if (rank==0){
FILE * log_file = fopen("Permeability.csv","a");
fprintf(log_file,"time Fx Fy Fz mu Vs As Js Xs vx vy vz k\n");
fclose(log_file);
bool WriteHeader=false;
FILE *log_file = fopen("Permeability.csv","r");
if (log_file != NULL)
fclose(log_file);
else
WriteHeader=true;
if (WriteHeader){
log_file = fopen("Permeability.csv","a+");
fprintf(log_file,"time Fx Fy Fz mu Vs As Js Xs vx vy vz k\n");
fclose(log_file);
}
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
starttime = MPI_Wtime();
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = Utilities::MPI::time();
if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax);
if (rank==0) printf("********************************************************\n");
timestep=0;
@ -215,18 +225,21 @@ void ScaLBL_MRTModel::Run(){
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier();
comm.barrier();
timestep++;
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier();
comm.barrier();
//************************************************************************/
if (timestep%1000==0){
ScaLBL_D3Q19_Momentum(fq,Velocity, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier();
comm.barrier();
ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x);
ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y);
ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z);
@ -248,10 +261,10 @@ void ScaLBL_MRTModel::Run(){
}
}
}
MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
vax = Mask->Comm.sumReduce( vax_loc );
vay = Mask->Comm.sumReduce( vay_loc );
vaz = Mask->Comm.sumReduce( vaz_loc );
count = Mask->Comm.sumReduce( count_loc );
vax /= count;
vay /= count;
@ -281,10 +294,10 @@ void ScaLBL_MRTModel::Run(){
double As = Morphology.A();
double Hs = Morphology.H();
double Xs = Morphology.X();
Vs=sumReduce( Dm->Comm, Vs);
As=sumReduce( Dm->Comm, As);
Hs=sumReduce( Dm->Comm, Hs);
Xs=sumReduce( Dm->Comm, Xs);
Vs = Dm->Comm.sumReduce( Vs);
As = Dm->Comm.sumReduce( As);
Hs = Dm->Comm.sumReduce( Hs);
Xs = Dm->Comm.sumReduce( Xs);
double h = Dm->voxel_length;
double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag;
if (rank==0) {
@ -297,7 +310,7 @@ void ScaLBL_MRTModel::Run(){
}
}
//************************************************************************/
stoptime = MPI_Wtime();
stoptime = Utilities::MPI::time();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
@ -318,7 +331,8 @@ void ScaLBL_MRTModel::VelocityField(){
/* Minkowski Morphology(Mask);
int SIZE=Np*sizeof(double);
ScaLBL_D3Q19_Momentum(fq,Velocity, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier();.
comm.barrier();
ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE);
memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double));
@ -345,10 +359,10 @@ void ScaLBL_MRTModel::VelocityField(){
vaz_loc += VELOCITY[2*Np+n];
count_loc+=1.0;
}
MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
vax = Mask->Comm.sumReduce( vax_loc );
vay = Mask->Comm.sumReduce( vay_loc );
vaz = Mask->Comm.sumReduce( vaz_loc );
count = Mask->Comm.sumReduce( count_loc );
vax /= count;
vay /= count;

6
models/MRTModel.h
View File

@ -11,13 +11,13 @@
#include "common/ScaLBL.h"
#include "common/Communication.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "analysis/Minkowski.h"
#include "ProfilerApp.h"
class ScaLBL_MRTModel{
public:
ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM);
ScaLBL_MRTModel(int RANK, int NP, const Utilities::MPI& COMM);
~ScaLBL_MRTModel();
// functions in they should be run
@ -63,7 +63,7 @@ public:
DoubleArray Velocity_y;
DoubleArray Velocity_z;
private:
MPI_Comm comm;
Utilities::MPI comm;
// filenames
char LocalRankString[8];

43
tests/BlobAnalyzeParallel.cpp
View File

@ -100,11 +100,10 @@ inline void WriteBlobStates(TwoPhase TCAT, double D, double porosity){
int main(int argc, char **argv)
{
// Initialize MPI
int rank, nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
Utilities::setAbortBehavior( true, true, true );
Utilities::setErrorHandlers();
PROFILE_ENABLE(0);
@ -137,20 +136,20 @@ int main(int argc, char **argv)
domain >> Ly;
domain >> Lz;
}
MPI_Barrier(comm);
comm.barrier();
// Computational domain
MPI_Bcast(&nx,1,MPI_INT,0,comm);
MPI_Bcast(&ny,1,MPI_INT,0,comm);
MPI_Bcast(&nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
comm.bcast(&nx,1,0);
comm.bcast(&ny,1,0);
comm.bcast(&nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// Check that the number of processors >= the number of ranks
if ( rank==0 ) {
@ -209,7 +208,7 @@ int main(int argc, char **argv)
// WriteLocalSolidID(LocalRankFilename, id, N);
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
ReadBinaryFile(LocalRankFilename, Averages.SDs.get(), N);
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
//.......................................................................
//copies of data needed to perform checkpointing from cpu
@ -221,7 +220,7 @@ int main(int argc, char **argv)
if (rank==0) printf("Reading restart file! \n");
// Read in the restart file to CPU buffers
ReadCheckpoint(LocalRestartFile, Den, DistEven, DistOdd, N);
MPI_Barrier(comm);
comm.barrier();
//.........................................................................
// Populate the arrays needed to perform averaging
if (rank==0) printf("Populate arrays \n");
@ -292,7 +291,7 @@ int main(int argc, char **argv)
}
Dm.CommInit(); // Initialize communications for domains
MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,comm);
sum_global = comm.sumReduce( sum );
porosity = sum_global/Dm.Volume;
if (rank==0) printf("Porosity = %f \n",porosity);
@ -329,14 +328,14 @@ int main(int argc, char **argv)
// BlobContainer Blobs;
DoubleArray RecvBuffer(dimx);
// MPI_Allreduce(&Averages.BlobAverages.get(),&Blobs.get(),1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Barrier(comm);
comm.barrier();
if (rank==0) printf("Number of components is %i \n",dimy);
for (int b=0; b<dimy; b++){
MPI_Allreduce(&Averages.BlobAverages(0,b),&RecvBuffer(0),dimx,MPI_DOUBLE,MPI_SUM,comm);
for (int idx=0; idx<dimx-1; idx++) Averages.BlobAverages(idx,b)=RecvBuffer(idx);
MPI_Barrier(comm);
comm.barrier();
if (Averages.BlobAverages(0,b) > 0.0){
double Vn,pn,awn,ans,Jwn,Kwn,lwns,cwns,trawn,trJwn;
@ -482,7 +481,7 @@ int main(int argc, char **argv)
fclose(BLOBS);*/
PROFILE_STOP("main");
PROFILE_SAVE("BlobIdentifyParallel",false);
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
return 0;
}

9
tests/BlobIdentifyParallel.cpp
View File

@ -47,11 +47,10 @@ void readRankData( int proc, int nx, int ny, int nz, DoubleArray& Phase, DoubleA
int main(int argc, char **argv)
{
// Initialize MPI
int rank, nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
#ifdef PROFILE
PROFILE_ENABLE(0);
PROFILE_DISABLE_TRACE();
@ -129,7 +128,7 @@ int main(int argc, char **argv)
PROFILE_STOP("main");
PROFILE_SAVE("BlobIdentifyParallel",false);
#endif
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
return 0;
}

9
tests/ColorToBinary.cpp
View File

@ -114,11 +114,10 @@ inline void ReadFromRank(char *FILENAME, DoubleArray &Phase, int nx, int ny, int
int main(int argc, char **argv)
{
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
printf("----------------------------------------------------------\n");
printf("Creating single Binary file from restart (8-bit integer)\n");
@ -276,7 +275,7 @@ int main(int argc, char **argv)
*/
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

9
tests/ComponentLabel.cpp
View File

@ -119,11 +119,10 @@ inline void ReadFromRank(char *FILENAME, DoubleArray &Phase, DoubleArray &Pressu
int main(int argc, char **argv)
{
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
printf("----------------------------------------------------------\n");
printf("COMPUTING TCAT ANALYSIS FOR NON-WETTING PHASE FEATURES \n");
@ -433,7 +432,7 @@ int main(int argc, char **argv)
fclose(DISTANCE);
*/
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

102
tests/GenerateSphereTest.cpp
View File

@ -9,7 +9,7 @@
//#include "common/pmmc.h"
#include "common/Domain.h"
#include "common/SpherePack.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
/*
@ -70,8 +70,8 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
}
}
// total Global is the number of nodes in the pore-space
MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm.Comm);
totalGlobal = Dm.Comm.sumReduce( count );
maxdistGlobal = Dm.Comm.sumReduce( maxdist );
double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2);
double porosity=totalGlobal/volume;
if (rank==0) printf("Media Porosity: %f \n",porosity);
@ -145,10 +145,9 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
// Increase the critical radius until the target saturation is met
double deltaR=0.05; // amount to change the radius in voxel units
double Rcrit_old;
double Rcrit_new;
double Rcrit_old=0.0;
double Rcrit_new=0.0;
double GlobalNumber = 1.f;
int imin,jmin,kmin,imax,jmax,kmax;
Rcrit_new = maxdistGlobal;
@ -214,42 +213,24 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
Dm.Comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag);
Dm.Comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag);
Dm.Comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag);
Dm.Comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag);
Dm.Comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag);
Dm.Comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag);
Dm.Comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag);
Dm.Comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag);
Dm.Comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag);
Dm.Comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag);
Dm.Comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag);
Dm.Comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag);
Dm.Comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag);
Dm.Comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag);
Dm.Comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag);
Dm.Comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag);
Dm.Comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag);
Dm.Comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag);
//......................................................................................
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);
@ -271,7 +252,7 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
UnpackID(Dm.recvList_YZ, Dm.recvCount_YZ ,recvID_YZ, id);
//......................................................................................
MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
//double GlobalNumber = Dm.Comm.sumReduce( LocalNumber );
count = 0.f;
for (int k=1; k<Nz-1; k++){
@ -284,7 +265,7 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
}
}
}
MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
countGlobal = Dm.Comm.sumReduce( count );
sw_new = countGlobal/totalGlobal;
sw_diff_new = abs(sw_new-SW);
// for test only
@ -314,15 +295,11 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
@ -363,7 +340,7 @@ int main(int argc, char **argv)
nspheres = domain_db->getScalar<int>( "nspheres");
//printf("Set domain \n");
int BoundaryCondition=1;
//int BoundaryCondition=1;
//Nz += 2;
//Nx = Ny = Nz; // Cubic domain
int N = Nx*Ny*Nz;
@ -396,7 +373,7 @@ int main(int argc, char **argv)
int sum = 0;
double sum_local;
double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs);
double porosity, pore_vol;
double porosity;
//...........................................................................
DoubleArray SignDist(Nx,Ny,Nz);
//.......................................................................
@ -412,14 +389,14 @@ int main(int argc, char **argv)
//.......................................................................
if (rank == 0) printf("Reading the sphere packing \n");
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
MPI_Barrier(comm);
comm.barrier();
// Broadcast the sphere packing to all processes
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
if (rank == 0){
// Compute the Sauter mean diameter
@ -433,7 +410,7 @@ int main(int argc, char **argv)
D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx;
printf("Sauter Mean Diameter (computed from sphere packing) = %f \n",D);
}
MPI_Bcast(&D,1,MPI_DOUBLE,0,comm);
comm.bcast(&D,1,0);
//.......................................................................
SignedDistance(SignDist.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz,
@ -450,7 +427,6 @@ int main(int argc, char **argv)
}
}
sum=0;
pore_vol = 0.0;
for ( k=1;k<Nz-1;k++){
for ( j=1;j<Ny-1;j++){
for ( i=1;i<Nx-1;i++){
@ -466,7 +442,7 @@ int main(int argc, char **argv)
}
}
sum_local = 1.0*sum;
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = comm.sumReduce(sum_local);
porosity = porosity*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -499,7 +475,7 @@ int main(int argc, char **argv)
//......................................................................
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

45
tests/TestBlobAnalyze.cpp
View File

@ -127,11 +127,10 @@ inline void WriteBlobStates(TwoPhase TCAT, double D, double porosity){
int main(int argc, char **argv)
{
// Initialize MPI
int rank, nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
if ( rank==0 ) {
@ -189,20 +188,20 @@ int main(int argc, char **argv)
Lx=Ly=Lz=1;
}
}
MPI_Barrier(comm);
comm.barrier();
// Computational domain
MPI_Bcast(&nx,1,MPI_INT,0,comm);
MPI_Bcast(&ny,1,MPI_INT,0,comm);
MPI_Bcast(&nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
comm.bcast(&nx,1,0);
comm.bcast(&ny,1,0);
comm.bcast(&nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// Check that the number of processors >= the number of ranks
if ( rank==0 ) {
@ -254,14 +253,14 @@ int main(int argc, char **argv)
cz[0]=0.25*Lz; cx[1]=0.75*Lz; cx[2]=0.25*Lz; cx[3]=0.25*Lz;
rad[0]=rad[1]=rad[2]=rad[3]=0.1*Lx;
MPI_Barrier(comm);
comm.barrier();
// Broadcast the sphere packing to all processes
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
//.......................................................................
SignedDistance(Averages.Phase.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz,
Dm->iproc(),Dm->jproc(),Dm->kproc(),Dm->nprocx(),Dm->nprocy(),Dm->nprocz());
@ -317,7 +316,7 @@ int main(int argc, char **argv)
delete [] rad;
} // Limit scope so variables that contain communicators will free before MPI_Finialize
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
return 0;
}

37
tests/TestBlobIdentify.cpp
View File

@ -23,21 +23,19 @@ inline double rand2()
// Test if all ranks agree on a value
bool allAgree( int x, MPI_Comm comm ) {
bool allAgree( int x, const Utilities::MPI& comm ) {
int x2 = x;
MPI_Bcast(&x2,1,MPI_INT,0,comm);
comm.bcast(&x2,1,0);
int diff = x==x2 ? 0:1;
int diff2 = 0;
MPI_Allreduce(&diff,&diff2,1,MPI_INT,MPI_SUM,comm);
int diff2 = comm.sumReduce( diff );
return diff2==0;
}
template<class T>
bool allAgree( const std::vector<T>& x, MPI_Comm comm ) {
bool allAgree( const std::vector<T>& x, const Utilities::MPI& comm ) {
std::vector<T> x2 = x;
MPI_Bcast(&x2[0],x.size()*sizeof(T)/sizeof(int),MPI_INT,0,comm);
comm.bcast(&x2[0],x.size()*sizeof(T)/sizeof(int),0);
int diff = x==x2 ? 0:1;
int diff2 = 0;
MPI_Allreduce(&diff,&diff2,1,MPI_INT,MPI_SUM,comm);
int diff2 = comm.sumReduce( diff );
return diff2==0;
}
@ -74,9 +72,9 @@ struct bubble_struct {
// Create a random set of bubles
std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly, double Lz, MPI_Comm comm )
std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly, double Lz, const Utilities::MPI& comm )
{
int rank = comm_rank(comm);
int rank = comm.getRank();
std::vector<bubble_struct> bubbles(N_bubbles);
if ( rank == 0 ) {
double R0 = 0.2*Lx*Ly*Lz/pow((double)N_bubbles,0.333);
@ -91,7 +89,7 @@ std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly,
}
}
size_t N_bytes = N_bubbles*sizeof(bubble_struct);
MPI_Bcast((char*)&bubbles[0],N_bytes,MPI_CHAR,0,comm);
comm.bcast((char*)&bubbles[0],N_bytes,0);
return bubbles;
}
@ -124,7 +122,7 @@ void fillBubbleData( const std::vector<bubble_struct>& bubbles, DoubleArray& Pha
// Shift all of the data by the given number of cells
void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct& rank_info, MPI_Comm comm )
void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct& rank_info, const Utilities::MPI& comm )
{
int nx = data.size(0)-2;
int ny = data.size(1)-2;
@ -154,11 +152,10 @@ void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct
int main(int argc, char **argv)
{
// Initialize MPI
int rank, nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
PROFILE_ENABLE(1);
PROFILE_DISABLE_TRACE();
PROFILE_SYNCHRONIZE();
@ -297,7 +294,7 @@ int main(int argc, char **argv)
velocity[i].z = bubbles[i].radius*(2*rand2()-1);
}
}
MPI_Bcast((char*)&velocity[0],bubbles.size()*sizeof(Point),MPI_CHAR,0,comm);
comm.bcast((char*)&velocity[0],bubbles.size()*sizeof(Point),0);
fillBubbleData( bubbles, Phase, SignDist, Lx, Ly, Lz, rank_info );
fillData.fill(Phase);
fillData.fill(SignDist);
@ -391,8 +388,8 @@ int main(int argc, char **argv)
printf("\n");
}
}
MPI_Bcast(&N1,1,MPI_INT,0,comm);
MPI_Bcast(&N2,1,MPI_INT,0,comm);
comm.bcast(&N1,1,0);
comm.bcast(&N2,1,0);
if ( N1!=nblobs || N2!=nblobs2 ) {
if ( rank==0 )
printf("Error, blob ids do not map in moving bubble test (%i,%i,%i,%i)\n",
@ -412,7 +409,7 @@ int main(int argc, char **argv)
// Finished
PROFILE_STOP("main");
PROFILE_SAVE("TestBlobIdentify",false);
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
return N_errors;
}

5
tests/TestBlobIdentifyCorners.cpp
View File

@ -18,10 +18,9 @@
int main(int argc, char **argv)
{
// Initialize MPI
int rank, nprocs;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
int rank = comm.getRank();
int nprocs = comm.getSize();
/*if ( nprocs != 8 ) {
printf("This tests requires 8 processors\n");
return -1;

87
tests/TestBubble.cpp
View File

@ -7,7 +7,7 @@
#include "analysis/pmmc.h"
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "IO/Mesh.h"
#include "IO/Writer.h"
@ -32,19 +32,19 @@ int main(int argc, char **argv)
// Initialize MPI
int provided_thread_support = -1;
MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support);
MPI_Comm comm;
MPI_Comm_dup(MPI_COMM_WORLD,&comm);
int rank = comm_rank(comm);
int nprocs = comm_size(comm);
if ( rank==0 && provided_thread_support<MPI_THREAD_MULTIPLE )
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
auto comm = Utilities::MPI( MPI_COMM_WORLD ).dup();
int rank = comm.getRank();
int nprocs = comm.getSize();
if ( rank==0 && provided_thread_support<MPI_THREAD_MULTIPLE )
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -119,7 +119,7 @@ int main(int argc, char **argv)
int jproc = rank_info.jy;
int kproc = rank_info.kz;
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
// **************************************************************
double Ps = -(das-dbs)/(das+dbs);
@ -162,7 +162,7 @@ int main(int argc, char **argv)
// Mask that excludes the solid phase
Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,pBC);
MPI_Barrier(comm);
comm.barrier();
Nx+=2; Ny+=2; Nz += 2;
@ -432,8 +432,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
//.........................................
//...........................................................................
// MAIN VARIABLES INITIALIZED HERE
@ -517,7 +517,7 @@ int main(int argc, char **argv)
ScaLBL_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
ScaLBL_CopyToDevice(Den,cDen,2*N*sizeof(double));
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
}
//*************************************************************************
@ -529,7 +529,7 @@ int main(int argc, char **argv)
ScaLBL_Comm.SendHalo(Phi);
ScaLBL_Comm.RecvHalo(Phi);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
if (rank==0 && pBC){
@ -560,7 +560,7 @@ int main(int argc, char **argv)
ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
ScaLBL_CopyToHost(Phase.data(),Phi,N*sizeof(double));
ScaLBL_CopyToHost(Press.data(),Pressure,N*sizeof(double));
MPI_Barrier(comm);
comm.barrier();
//...........................................................................
int timestep = 0;
@ -591,7 +591,7 @@ int main(int argc, char **argv)
//*************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
// Swap the distributions for momentum transport
//*************************************************************************
@ -599,7 +599,7 @@ int main(int argc, char **argv)
//*************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
// Wait for communications to complete and unpack the distributions
ScaLBL_Comm.RecvD3Q19(f_even, f_odd);
@ -616,7 +616,7 @@ int main(int argc, char **argv)
ScaLBL_D3Q7_Swap(ID, B_even, B_odd, Nx, Ny, Nz);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
// Wait for communication and unpack the D3Q7 distributions
@ -633,7 +633,7 @@ int main(int argc, char **argv)
//*************************************************************************
// ScaLBL_ComputePhaseField(ID, Phi, Copy, Den, N);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
ScaLBL_ComputePhaseField(ID, Phi, Den, N);
//*************************************************************************
@ -659,7 +659,7 @@ int main(int argc, char **argv)
//...................................................................................
MPI_Barrier(comm);
comm.barrier();
// Timestep completed!
timestep++;
@ -807,27 +807,27 @@ int main(int argc, char **argv)
//...........................................................................
}
//...........................................................................
MPI_Barrier(comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
comm.barrier();
nwp_volume_global = comm.sumReduce( nwp_volume );
awn_global = comm.sumReduce( awn );
ans_global = comm.sumReduce( ans );
aws_global = comm.sumReduce( aws );
lwns_global = comm.sumReduce( lwns );
As_global = comm.sumReduce( As );
Jwn_global = comm.sumReduce( Jwn );
efawns_global = comm.sumReduce( efawns );
// Phase averages
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Barrier(comm);
vol_w_global = comm.sumReduce( vol_w );
vol_n_global = comm.sumReduce( vol_n );
paw_global = comm.sumReduce( paw );
pan_global = comm.sumReduce( pan );
vaw_global(0) = comm.sumReduce( vaw(0) );
van_global(0) = comm.sumReduce( van(0) );
vawn_global(0) = comm.sumReduce( vawn(0) );
Gwn_global(0) = comm.sumReduce( Gwn(0) );
Gns_global(0) = comm.sumReduce( Gns(0) );
Gws_global(0) = comm.sumReduce( Gws(0) );
comm.barrier();
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
// See McClure, Prins and Miller (2013)
@ -950,8 +950,8 @@ int main(int argc, char **argv)
//************************************************************************/
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
@ -989,9 +989,8 @@ int main(int argc, char **argv)
PROFILE_SAVE("TestBubble");
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
} // Limit scope so variables that contain communicators will free before MPI_Finialize
MPI_Comm_free(&comm);
MPI_Finalize();
return 0;
}

74
tests/TestBubbleDFH.cpp
View File

@ -9,7 +9,7 @@
#include "common/Communication.h"
#include "analysis/TwoPhase.h"
#include "analysis/runAnalysis.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
@ -29,10 +29,9 @@ int main(int argc, char **argv)
// Initialize MPI
int provided_thread_support = -1;
MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support);
MPI_Comm comm;
MPI_Comm_dup(MPI_COMM_WORLD,&comm);
int rank = comm_rank(comm);
int nprocs = comm_size(comm);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check=0;
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
int i,j,k,n,Np;
@ -45,7 +44,7 @@ int main(int argc, char **argv)
int device=ScaLBL_SetDevice(rank);
printf("Using GPU ID %i for rank %i \n",device,rank);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
PROFILE_ENABLE(1);
//PROFILE_ENABLE_TRACE();
@ -72,13 +71,8 @@ int main(int argc, char **argv)
// Initialize compute device
// int device=ScaLBL_SetDevice(rank);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
PROFILE_ENABLE(1);
//PROFILE_ENABLE_TRACE();
//PROFILE_ENABLE_MEMORY();
PROFILE_SYNCHRONIZE();
PROFILE_START("Main");
Utilities::setErrorHandlers();
// Variables that specify the computational domain
@ -123,7 +117,7 @@ int main(int argc, char **argv)
// Get the rank info
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
MPI_Barrier(comm);
comm.barrier();
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
@ -164,15 +158,15 @@ int main(int argc, char **argv)
pBC=false;
// Full domain used for averaging (do not use mask for analysis)
std::shared_ptr<Domain> Dm(new Domain(domain_db,comm));
auto Dm = std::make_shared<Domain>(domain_db,comm);
for (int i=0; i<Dm->Nx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1;
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
auto Averages = std::make_shared<TwoPhase>(Dm);
// TwoPhase Averages(Dm);
Dm->CommInit();
// Mask that excludes the solid phase
std::shared_ptr<Domain> Mask(new Domain(domain_db,comm));
MPI_Barrier(comm);
auto Mask = std::make_shared<Domain>(domain_db,comm);
comm.barrier();
Nx+=2; Ny+=2; Nz += 2;
int N = Nx*Ny*Nz;
@ -191,9 +185,8 @@ int main(int argc, char **argv)
// printf("Local File Name = %s \n",LocalRankFilename);
// .......... READ THE INPUT FILE .......................................
// char value;
char *id;
id = new char[N];
double sum, sum_local;
auto id = new char[N];
double sum;
//...........................................................................
if (rank == 0) cout << "Setting up bubble..." << endl;
double BubbleRadius = 15.5; // Radius of the capillary tube
@ -244,21 +237,19 @@ int main(int argc, char **argv)
// Initialize communication structures in averaging domain
for (i=0; i<Mask->Nx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i];
Mask->CommInit();
double *PhaseLabel;
PhaseLabel = new double[N];
auto PhaseLabel = new double[N];
//...........................................................................
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device (will use optimized layout)
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm(new ScaLBL_Communicator(Mask));
auto ScaLBL_Comm = std::make_shared<ScaLBL_Communicator>(Mask);
int Npad=(Np/16 + 2)*16;
if (rank==0) printf ("Set up memory efficient layout Npad=%i \n",Npad);
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
MPI_Barrier(comm);
comm.barrier();
//...........................................................................
// MAIN VARIABLES ALLOCATED HERE
@ -395,8 +386,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
//.........................................
err = 1.0;
@ -445,7 +436,7 @@ int main(int argc, char **argv)
}
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
// *************EVEN TIMESTEP*************
timestep++;
@ -481,9 +472,9 @@ int main(int argc, char **argv)
}
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
//************************************************************************
MPI_Barrier(comm);
comm.barrier();
PROFILE_STOP("Update");
// Run the analysis
@ -495,8 +486,8 @@ int main(int argc, char **argv)
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
comm.barrier();
stoptime = Utilities::MPI::time();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
@ -515,9 +506,8 @@ int main(int argc, char **argv)
// Copy back final phase indicator field and convert to regular layout
DoubleArray PhaseField(Nx,Ny,Nz);
ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField);
FILE *OUTFILE;
sprintf(LocalRankFilename,"Phase.raw",rank);
OUTFILE = fopen(LocalRankFilename,"wb");
sprintf(LocalRankFilename,"Phase.raw");
auto OUTFILE = fopen(LocalRankFilename,"wb");
fwrite(PhaseField.data(),8,N,OUTFILE);
fclose(OUTFILE);
@ -535,9 +525,8 @@ int main(int argc, char **argv)
}
}
}
FILE *GFILE;
sprintf(LocalRankFilename,"Gradient.raw");
GFILE = fopen(LocalRankFilename,"wb");
auto GFILE = fopen(LocalRankFilename,"wb");
fwrite(GradNorm.data(),8,N,GFILE);
fclose(GFILE);
@ -545,23 +534,20 @@ int main(int argc, char **argv)
DoubleArray Rho2(Nx,Ny,Nz);
ScaLBL_Comm->RegularLayout(Map,&Den[0],Rho1);
ScaLBL_Comm->RegularLayout(Map,&Den[Np],Rho2);
FILE *RFILE1;
sprintf(LocalRankFilename,"Rho1.raw");
RFILE1 = fopen(LocalRankFilename,"wb");
auto RFILE1 = fopen(LocalRankFilename,"wb");
fwrite(Rho1.data(),8,N,RFILE1);
fclose(RFILE1);
FILE *RFILE2;
sprintf(LocalRankFilename,"Rho2.raw");
RFILE2 = fopen(LocalRankFilename,"wb");
auto RFILE2 = fopen(LocalRankFilename,"wb");
fwrite(Rho2.data(),8,N,RFILE2);
fclose(RFILE2);
PROFILE_STOP("Main");
PROFILE_SAVE("lbpm_color_simulator",1);
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
} // Limit scope so variables that contain communicators will free before MPI_Finialize
MPI_Comm_free(&comm);
MPI_Finalize();
return check;
}

14
tests/TestColorBubble.cpp
View File

@ -7,7 +7,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "models/ColorModel.h"
using namespace std;
@ -64,15 +64,11 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius)
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check=0;
{
if (rank == 0){
@ -97,7 +93,7 @@ int main(int argc, char **argv)
ColorModel.WriteDebug();
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************

44
tests/TestColorGrad.cpp
View File

@ -7,7 +7,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -15,15 +15,11 @@ using namespace std;
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check;
{
// parallel domain size (# of sub-domains)
@ -116,20 +112,20 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
// **************************************************************
@ -146,7 +142,7 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
MPI_Barrier(comm);
comm.barrier();
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
@ -175,7 +171,7 @@ int main(int argc, char **argv)
}
}
Dm.CommInit();
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
if (rank==0) printf ("Create ScaLBL_Communicator \n");
@ -192,7 +188,7 @@ int main(int argc, char **argv)
neighborList= new int[18*Np];
ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
MPI_Barrier(comm);
comm.barrier();
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
@ -260,7 +256,7 @@ int main(int argc, char **argv)
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************

24
tests/TestColorGradDFH.cpp
View File

@ -7,7 +7,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -25,15 +25,11 @@ std::shared_ptr<Database> loadInputs( int nprocs )
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check=0;
{
// parallel domain size (# of sub-domains)
@ -53,9 +49,6 @@ int main(int argc, char **argv)
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
int nprocx = db->getVector<int>( "nproc" )[0];
int nprocy = db->getVector<int>( "nproc" )[1];
int nprocz = db->getVector<int>( "nproc" )[2];
if (rank==0){
printf("********************************************************\n");
@ -64,7 +57,7 @@ int main(int argc, char **argv)
}
// Get the rank info
std::shared_ptr<Domain> Dm(new Domain(db,comm));
auto Dm = std::make_shared<Domain>(db,comm);
Nx += 2;
Ny += 2;
Nz += 2;
@ -85,7 +78,7 @@ int main(int argc, char **argv)
}
}
Dm->CommInit();
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
if (rank==0) printf ("Create ScaLBL_Communicator \n");
@ -108,10 +101,9 @@ int main(int argc, char **argv)
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
comm.barrier();
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
int neighborSize=18*Np*sizeof(int);
if (rank==0) printf ("Allocating distributions \n");
int *NeighborList;
@ -215,7 +207,7 @@ int main(int argc, char **argv)
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************

32
tests/TestColorMassBounceback.cpp
View File

@ -7,7 +7,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -15,15 +15,11 @@ using namespace std;
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check=0;
{
// parallel domain size (# of sub-domains)
@ -42,7 +38,7 @@ int main(int argc, char **argv)
// Initialize compute device
// int device=ScaLBL_SetDevice(rank);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
Utilities::setErrorHandlers();
// Variables that specify the computational domain
@ -77,7 +73,7 @@ int main(int argc, char **argv)
// Get the rank info
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
MPI_Barrier(comm);
comm.barrier();
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
@ -121,7 +117,7 @@ int main(int argc, char **argv)
std::shared_ptr<Domain> Dm(new Domain(domain_db,comm));
for (int i=0; i<Dm->Nx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1;
Dm->CommInit();
MPI_Barrier(comm);
comm.barrier();
Nx+=2; Ny+=2; Nz += 2;
int N = Nx*Ny*Nz;
@ -153,7 +149,7 @@ int main(int argc, char **argv)
}
}
Dm->CommInit();
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
if (rank==0) printf ("Create ScaLBL_Communicator \n");
@ -170,7 +166,7 @@ int main(int argc, char **argv)
Npad=Np+32;
neighborList= new int[18*Npad];
Np=ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
comm.barrier();
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
@ -272,7 +268,7 @@ int main(int argc, char **argv)
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
@ -332,7 +328,7 @@ int main(int argc, char **argv)
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
//************************************************************************
printf("Check after even time \n");
@ -415,7 +411,7 @@ int main(int argc, char **argv)
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
@ -476,7 +472,7 @@ int main(int argc, char **argv)
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
//************************************************************************
printf("Check after even time \n");
@ -523,7 +519,7 @@ int main(int argc, char **argv)
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
return check;

14
tests/TestColorSquareTube.cpp
View File

@ -7,7 +7,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "models/ColorModel.h"
std::shared_ptr<Database> loadInputs( int nprocs )
@ -84,15 +84,11 @@ void InitializeSquareTube(ScaLBL_ColorModel &ColorModel){
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check=0;
{
if (rank == 0){
@ -113,7 +109,7 @@ int main(int argc, char **argv)
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************

54
tests/TestCommD3Q19.cpp
View File

@ -6,7 +6,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -49,7 +49,7 @@ extern void GlobalFlipScaLBL_D3Q19_Init(double *dist, IntArray Map, int Np, int
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
int q,i,j,k,n,N;
int q,i,j,k;
int Cqx,Cqy,Cqz; // Discrete velocity
int x,y,z; // Global indices
int xn,yn,zn; // Global indices of neighbor
@ -59,8 +59,6 @@ extern void GlobalFlipScaLBL_D3Q19_Init(double *dist, IntArray Map, int Np, int
Y = Ny*nprocy;
Z = Nz*nprocz;
NULL_USE(Z);
N = (Nx+2)*(Ny+2)*(Nz+2); // size of the array including halo
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
@ -104,16 +102,13 @@ extern int GlobalCheckDebugDist(double *dist, IntArray Map, int Np, int Nx, int
{
int returnValue = 0;
int q,i,j,k,n,N,idx;
int Cqx,Cqy,Cqz; // Discrete velocity
int q,i,j,k,idx;
int x,y,z; // Global indices
int xn,yn,zn; // Global indices of neighbor
int X,Y,Z; // Global size
X = Nx*nprocx;
Y = Ny*nprocy;
Z = Nz*nprocz;
NULL_USE(Z);
N = (Nx+2)*(Ny+2)*(Nz+2); // size of the array including halo
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
@ -168,20 +163,13 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check;
{
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -191,11 +179,8 @@ int main(int argc, char **argv)
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double tau,Fx,Fy,Fz,tol;
// Domain variables
int i,j,k,n;
int i,j,k;
// Load inputs
auto db = loadInputs( nprocs );
@ -223,8 +208,7 @@ int main(int argc, char **argv)
char LocalRankFilename[40];
sprintf(LocalRankFilename,"ID.%05i",rank);
char *id;
id = new char[Nx*Ny*Nz];
auto id = new char[Nx*Ny*Nz];
/* if (rank==0) printf("Assigning phase ID from file \n");
if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n");
@ -237,7 +221,7 @@ int main(int argc, char **argv)
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny+j*Nx+i;
int n = k*Nx*Ny+j*Nx+i;
id[n] = 1;
Dm->id[n] = id[n];
}
@ -270,7 +254,7 @@ int main(int argc, char **argv)
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
int n = k*Nx*Ny+j*Nx+i;
if (id[n] == component){
sum_local+=1.0;
}
@ -278,14 +262,14 @@ int main(int argc, char **argv)
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
sum = comm.sumReduce( sum_local );
double iVol_global=1.f/double((Nx-2)*(Ny-2)*(Nz-2)*nprocx*nprocy*nprocz);
porosity = 1.0-sum*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
//.......................................................................
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
@ -300,7 +284,7 @@ int main(int argc, char **argv)
IntArray Map(Nx,Ny,Nz);
Map.fill(-2);
Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
comm.barrier();
int neighborSize=18*Np*sizeof(int);
//......................device distributions.................................
dist_mem_size = Np*sizeof(double);
@ -370,7 +354,7 @@ int main(int argc, char **argv)
GlobalFlipScaLBL_D3Q19_Init(fq_host, Map, Np, Nx-2, Ny-2, Nz-2, iproc,jproc,kproc,nprocx,nprocy,nprocz);
ScaLBL_CopyToDevice(fq, fq_host, 19*dist_mem_size);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
// First timestep
ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
@ -393,8 +377,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
starttime = MPI_Wtime();
comm.barrier();
starttime = Utilities::MPI::time();
//.........................................
@ -413,13 +397,13 @@ int main(int argc, char **argv)
//*********************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
// Iteration completed!
timestep++;
//...................................................................
}
//************************************************************************/
stoptime = MPI_Wtime();
stoptime = Utilities::MPI::time();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -442,7 +426,7 @@ int main(int argc, char **argv)
if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9);
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************

9
tests/TestDatabase.cpp
View File

@ -9,7 +9,7 @@
#include "common/UnitTest.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Database.h"
#include "ProfilerApp.h"
@ -17,11 +17,8 @@
// Main
int main(int argc, char **argv)
{
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
Utilities::setAbortBehavior(true,2);
Utilities::setErrorHandlers();
UnitTest ut;
@ -69,7 +66,7 @@ int main(int argc, char **argv)
// Finished
PROFILE_SAVE("TestDatabase",true);
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
return err;
}

38
tests/TestFluxBC.cpp
View File

@ -1,5 +1,5 @@
#include <iostream>
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "common/ScaLBL.h"
@ -18,9 +18,9 @@ std::shared_ptr<Database> loadInputs( int nprocs )
int main (int argc, char **argv)
{
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
int rank = MPI_WORLD_RANK();
int nprocs = MPI_WORLD_SIZE();
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
// set the error code
// Note: the error code should be consistent across all processors
@ -32,21 +32,14 @@ int main (int argc, char **argv)
}
{
int i,j,k,n,Np;
bool pBC=true;
double Lx,Ly,Lz;
Lx = Ly = Lz = 1.f;
double din,dout;
int BC=1;
// Load inputs
auto db = loadInputs( nprocs );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
int nprocx = db->getVector<int>( "nproc" )[0];
int nprocy = db->getVector<int>( "nproc" )[1];
int nprocz = db->getVector<int>( "nproc" )[2];
std::shared_ptr<Domain> Dm(new Domain(db,comm));
auto Dm = std::make_shared<Domain>(db,comm);
Nx += 2; Ny+=2; Nz += 2;
Nx = Ny = Nz; // Cubic domain
@ -55,8 +48,7 @@ int main (int argc, char **argv)
//.......................................................................
// Assign the phase ID
//.......................................................................
char *id;
id = new char[N];
auto id = new char[N];
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
@ -97,7 +89,7 @@ int main (int argc, char **argv)
neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
comm.barrier();
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
@ -157,12 +149,10 @@ int main (int argc, char **argv)
double *VEL;
VEL= new double [3*Np];
int SIZE=3*Np*sizeof(double);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
ScaLBL_CopyToHost(&VEL[0],&dvc_vel[0],SIZE);
double err,value,Q;
Q = 0.f;
double Q = 0.f;
k=1;
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
@ -176,7 +166,7 @@ int main (int argc, char **argv)
// respect backwards read / write!!!
printf("Inlet Flux: input=%f, output=%f \n",flux,Q);
err = fabs(flux + Q);
double err = fabs(flux + Q);
if (err > 1e-12){
error = 1;
printf(" Inlet error %f \n",err);
@ -185,7 +175,7 @@ int main (int argc, char **argv)
// Consider a larger number of timesteps and simulate flow
double Fx, Fy, Fz;
double tau = 1.0;
double mu=(tau-0.5)/3.0;
//double mu=(tau-0.5)/3.0;
double rlx_setA=1.0/tau;
double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
dout=1.f;
@ -202,7 +192,7 @@ int main (int argc, char **argv)
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
@ -211,7 +201,7 @@ int main (int argc, char **argv)
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
//************************************************************************/
@ -275,7 +265,7 @@ int main (int argc, char **argv)
}
// Finished
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
return error;
}

25
tests/TestForceD3Q19.cpp
View File

@ -1,5 +1,5 @@
#include <iostream>
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <math.h>
@ -443,13 +443,14 @@ inline void MRT_Transform(double *dist, int Np, double Fx, double Fy, double Fz)
int main (int argc, char **argv)
{
MPI_Init(&argc,&argv);
int rank = MPI_WORLD_RANK();
int nprocs = MPI_WORLD_SIZE();
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
for (int i=0; i<nprocs; i++) {
if ( rank==i )
printf("%i of %i: Testing force term \n",rank,nprocs);
MPI_Barrier(MPI_COMM_WORLD);
comm.barrier();
}
// Create a memory leak for valgrind to find
@ -457,24 +458,16 @@ int main (int argc, char **argv)
double *x = new double[1];
ASSERT(x!=NULL);
}
// set the error code
// Note: the error code should be consistent across all processors
int error = 0;
int Np = 1;
int Q = 9;
//int Q = 9;
double Fx = 1.0;
double Fy = 1.0;
double Fz = 1.0;
double *dist;
double * Velocity;
dist = new double [19*Np];
Velocity = new double [3*Np];
auto dist = new double [19*Np];
//auto Velocity = new double [3*Np
for (int n=0; n<Np; n++){
dist[n] = 0.3333333333333333;
@ -547,6 +540,6 @@ int main (int argc, char **argv)
printf("Fy = %f; Computed vy=%f \n",Fy,vel[Np+0]);
printf("Fz = %f; Computed vz=%f \n",Fz,vel[2*Np+0]);
MPI_Barrier(MPI_COMM_WORLD);
comm.barrier();
MPI_Finalize();
}

34
tests/TestForceMoments.cpp
View File

@ -7,7 +7,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -46,15 +46,11 @@ std::shared_ptr<Database> loadInputs( int nprocs )
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check=0;
{
// parallel domain size (# of sub-domains)
@ -98,7 +94,7 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
MPI_Barrier(comm);
comm.barrier();
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -106,7 +102,7 @@ int main(int argc, char **argv)
if (rank == 0) {
printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
}
MPI_Barrier(comm);
comm.barrier();
if (rank == 1){
printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
printf("\n\n");
@ -143,7 +139,7 @@ int main(int argc, char **argv)
}
}
Dm->CommInit();
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
int Np=0; // number of local pore nodes
@ -188,7 +184,7 @@ int main(int argc, char **argv)
if (rank == 0) PrintNeighborList(neighborList,Np, rank);
MPI_Barrier(comm);
comm.barrier();
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
@ -213,13 +209,13 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
starttime = MPI_Wtime();
ScaLBL_DeviceBarrier(); comm.barrier();
starttime = Utilities::MPI::time();
/************ MAIN ITERATION LOOP (timing communications)***************************************/
//ScaLBL_Comm->SendD3Q19(dist, &dist[10*Np]);
//ScaLBL_Comm->RecvD3Q19(dist, &dist[10*Np]);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
if (rank==0) printf("Beginning AA timesteps...\n");
if (rank==0) printf("********************************************************\n");
@ -231,14 +227,14 @@ int main(int argc, char **argv)
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
ScaLBL_Comm->SendD3Q19AA(dist); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
//************************************************************************/
@ -248,7 +244,7 @@ int main(int argc, char **argv)
//************************************************************************/
stoptime = MPI_Wtime();
stoptime = Utilities::MPI::time();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -331,7 +327,7 @@ int main(int argc, char **argv)
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************

36
tests/TestInterfaceSpeed.cpp
View File

@ -2,7 +2,7 @@
#include <math.h>
#include "analysis/TwoPhase.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "IO/Mesh.h"
#include "IO/Writer.h"
@ -18,13 +18,9 @@
int main (int argc, char *argv[])
{
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int i,j,k,n;
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
// Load inputs
string FILENAME = argv[1];
@ -36,24 +32,24 @@ int main (int argc, char *argv[])
int Ny = domain_db->getVector<int>( "n" )[1];
int Nz = domain_db->getVector<int>( "n" )[2];
std::shared_ptr<Domain> Dm(new Domain(domain_db,comm));
auto Dm = std::make_shared<Domain>(domain_db,comm);
Nx+=2; Ny+=2; Nz+=2;
for (i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;
Dm->CommInit();
std::shared_ptr<TwoPhase> Averages(new TwoPhase(Dm));
auto Averages = std::make_shared<TwoPhase>(Dm);
int timestep=0;
double Cx,Cy,Cz;
double dist1,dist2;
Cx = Cy = Cz = N*0.5;
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
for (int i=0; i<Nx; i++){
dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
dist2 = fabs(Cz-k)-HEIGHT;
@ -62,9 +58,9 @@ int main (int argc, char *argv[])
}
}
Cz += SPEED;
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
for (int i=0; i<Nx; i++){
dist1 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)) - RADIUS;
dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
@ -77,9 +73,9 @@ int main (int argc, char *argv[])
}
}
Cz += SPEED;
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
for (int i=0; i<Nx; i++){
dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
dist2 = fabs(Cz-k)-HEIGHT;
@ -151,7 +147,7 @@ int main (int argc, char *argv[])
return toReturn;
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
return 0;
MPI_Finalize();
// ****************************************************

66
tests/TestMRT.cpp
View File

@ -7,7 +7,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -488,15 +488,11 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
int check;
{
// parallel domain size (# of sub-domains)
@ -582,20 +578,20 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
comm.barrier();
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
//.................................................
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
// **************************************************************
@ -613,7 +609,7 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
MPI_Barrier(comm);
comm.barrier();
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -621,7 +617,7 @@ int main(int argc, char **argv)
if (rank == 0) {
printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
}
MPI_Barrier(comm);
comm.barrier();
if (rank == 1){
printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
printf("\n\n");
@ -650,7 +646,7 @@ int main(int argc, char **argv)
fread(Dm.id,1,N,IDFILE);
fclose(IDFILE);
MPI_Barrier(comm);
comm.barrier();
Dm.CommInit();
//.......................................................................
@ -671,12 +667,12 @@ int main(int argc, char **argv)
}
}
}
MPI_Barrier(comm);
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
comm.barrier();
sum = comm.sumReduce( sum_local );
porosity = sum*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
MPI_Barrier(comm);
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
if (rank==0) printf ("Create ScaLBL_Communicator \n");
@ -706,7 +702,7 @@ int main(int argc, char **argv)
neighborList= new int[18*Np];
ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
MPI_Barrier(comm);
comm.barrier();
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
@ -734,8 +730,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
starttime = MPI_Wtime();
ScaLBL_DeviceBarrier(); comm.barrier();
starttime = Utilities::MPI::time();
while (timestep < timesteps) {
@ -743,20 +739,20 @@ int main(int argc, char **argv)
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm.next, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
ScaLBL_Comm.SendD3Q19AA(dist); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm.next, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
timestep++;
//************************************************************************/
}
//************************************************************************/
stoptime = MPI_Wtime();
stoptime = Utilities::MPI::time();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -783,7 +779,7 @@ int main(int argc, char **argv)
VEL= new double [3*Np];
int SIZE=3*Np*sizeof(double);
ScaLBL_D3Q19_Momentum(dist,Velocity, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_DeviceBarrier(); comm.barrier();
ScaLBL_CopyToHost(&VEL[0],&Velocity[0],SIZE);
sum_local=0.f;
@ -799,13 +795,13 @@ int main(int argc, char **argv)
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
sum = comm.sumReduce( sum_local );
double PoreVel = sum*iVol_global;
if (rank==0) printf("Velocity = %f \n",PoreVel);
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************

23
tests/TestMap.cpp
View File

@ -7,7 +7,7 @@
#include <iostream>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
using namespace std;
@ -26,15 +26,9 @@ std::shared_ptr<Database> loadInputs( int nprocs )
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int check=0;
{
@ -45,6 +39,7 @@ int main(int argc, char **argv)
{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
int rank = comm.getRank();
if (rank == 0){
printf("********************************************************\n");
printf("Running unit test: TestMap \n");
@ -52,15 +47,11 @@ int main(int argc, char **argv)
}
// Load inputs
auto db = loadInputs( nprocs );
auto db = loadInputs( comm.getSize() );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
int nprocx = db->getVector<int>( "nproc" )[0];
int nprocy = db->getVector<int>( "nproc" )[1];
int nprocz = db->getVector<int>( "nproc" )[2];
std::shared_ptr<Domain> Dm(new Domain(db,comm));
auto Dm = std::make_shared<Domain>(db,comm);
Nx += 2;
Ny += 2;
@ -98,7 +89,7 @@ int main(int argc, char **argv)
neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
comm.barrier();
// Check the neighborlist
printf("Check neighborlist: exterior %i, first interior %i last interior %i \n",ScaLBL_Comm->LastExterior(),ScaLBL_Comm->FirstInterior(),ScaLBL_Comm->LastInterior());
@ -201,7 +192,7 @@ int main(int argc, char **argv)
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************

36
tests/TestMassConservationD3Q7.cpp
View File

@ -8,7 +8,7 @@
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "models/ColorModel.h"
inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius){
@ -66,29 +66,12 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius)
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
int sendtag,recvtag;
//*****************************************
// MPI ranks for all 18 neighbors
//**********************************
int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z;
int rank_xy,rank_XY,rank_xY,rank_Xy;
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -110,7 +93,6 @@ int main(int argc, char **argv)
Ny = CM.Ny;
Nz = CM.Nz;
N = Nx*Ny*Nz;
int dist_mem_size = N*sizeof(double);
//CM.ReadInput();
double radius=0.4*double(Nx);
@ -142,11 +124,9 @@ int main(int argc, char **argv)
CM.Run();
int D3Q7[7][3]={{0,0,0},{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}};
// Compare and make sure mass is conserved at every lattice site
double *Error;
Error = new double [N];
double *A_q, *B_q;
A_q = new double [7*Np];
B_q = new double [7*Np];
auto Error = new double[N];
auto A_q = new double[7*Np];
//auto B_q = new double[7*Np];
bool CleanCheck = true;
double original,final, sum_q;
double total_mass_A_0 = 0.0;
@ -285,7 +265,7 @@ int main(int argc, char **argv)
}
}
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
MPI_Finalize();
// ****************************************************
}

11
tests/TestMicroCTReader.cpp
View File

@ -1,6 +1,6 @@
// Test reading high-resolution files from the microct database
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/UnitTest.h"
#include "common/Database.h"
#include "common/Domain.h"
@ -13,12 +13,14 @@
void testReadMicroCT( const std::string& filename, UnitTest& ut )
{
Utilities::MPI comm( MPI_COMM_WORLD );
// Get the domain info
auto db = std::make_shared<Database>( filename );
auto domain_db = db->getDatabase( "Domain" );
// Test reading microCT files
auto data = readMicroCT( *domain_db, MPI_COMM_WORLD );
auto data = readMicroCT( *domain_db, comm );
// Check if we loaded the data correctly
if ( data.size() == domain_db->getVector<size_t>( "n" ) )
@ -30,7 +32,7 @@ void testReadMicroCT( const std::string& filename, UnitTest& ut )
auto n = domain_db->getVector<int>( "n" );
auto nproc = domain_db->getVector<int>( "nproc" );
int N[3] = { n[0]*nproc[0], n[1]*nproc[1], n[2]*nproc[2] };
int rank = comm_rank(MPI_COMM_WORLD);
int rank = comm.getRank();
RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] );
std::vector<IO::MeshDataStruct> meshData( 1 );
auto Var = std::make_shared<IO::Variable>();
@ -41,7 +43,7 @@ void testReadMicroCT( const std::string& filename, UnitTest& ut )
meshData[0].meshName = "grid";
meshData[0].mesh = std::make_shared<IO::DomainMesh>(rankInfo,n[0],n[1],n[2],N[0],N[1],N[2]);
meshData[0].vars.push_back(Var);
IO::writeData( 0, meshData, MPI_COMM_WORLD );
IO::writeData( 0, meshData, comm );
}
@ -60,7 +62,6 @@ int main(int argc, char **argv)
int N_errors = ut.NumFailGlobal();
// Close MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return N_errors;
}

11
tests/TestMomentsD3Q19.cpp
View File

@ -1,5 +1,5 @@
#include <iostream>
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <math.h>
@ -463,13 +463,14 @@ inline void MRT_Transform(double *dist, int Np) {
int main (int argc, char **argv)
{
MPI_Init(&argc,&argv);
int rank = MPI_WORLD_RANK();
int nprocs = MPI_WORLD_SIZE();
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
for (int i=0; i<nprocs; i++) {
if ( rank==i )
printf("%i of %i: TestMoments\n",rank,nprocs);
MPI_Barrier(MPI_COMM_WORLD);
comm.barrier();
}
// Create a memory leak for valgrind to find
@ -538,7 +539,7 @@ int main (int argc, char **argv)
error=count;
// Finished
MPI_Barrier(MPI_COMM_WORLD);
comm.barrier();
MPI_Finalize();
return error;
}

Some files were not shown because too many files have changed in this diff Show More