Replacing MPI_COMM_WORLD with a passed communicator
This commit is contained in:
Mark Berrill
@@ -156,16 +156,16 @@ static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
|
||||
|
||||
|
||||
// Write the mesh data
|
||||
void IO::writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, int format )
|
||||
void IO::writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, int format, MPI_Comm comm )
|
||||
{
|
||||
PROFILE_START("writeData");
|
||||
int rank = MPI_WORLD_RANK();
|
||||
int rank = comm_rank(comm);
|
||||
// Create the output directory
|
||||
char path[100];
|
||||
sprintf(path,"vis%03i",timestep);
|
||||
if ( rank == 0 )
|
||||
mkdir(path,S_IRWXU|S_IRGRP);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Write the mesh files
|
||||
std::vector<IO::MeshDatabase> meshes_written;
|
||||
if ( format == 1 ) {
|
||||
@@ -178,7 +178,7 @@ void IO::writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshDat
|
||||
ERROR("Unknown format");
|
||||
}
|
||||
// Gather a complete list of files on rank 0
|
||||
meshes_written = gatherAll(meshes_written,MPI_COMM_WORLD);
|
||||
meshes_written = gatherAll(meshes_written,comm);
|
||||
// Write the summary files
|
||||
if ( rank == 0 ) {
|
||||
// Write the summary file for the current timestep
|
||||
|
||||
@@ -21,8 +21,9 @@ namespace IO {
|
||||
* 1 - Old mesh format (provided for backward compatibility, cannot write variables)
|
||||
* 2 - New format, 1 file/process, double precision
|
||||
* 3 - New format, 1 file/process, single precision (not finished)
|
||||
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
|
||||
*/
|
||||
void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, int format );
|
||||
void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, int format, MPI_Comm comm );
|
||||
|
||||
|
||||
} // IO namespace
|
||||
|
||||
@@ -186,7 +186,7 @@ int ComputeLocalPhaseComponent(const IntArray &PhaseID, int &VALUE, BlobIDArray
|
||||
/******************************************************************
|
||||
* Reorder the global blob ids *
|
||||
******************************************************************/
|
||||
static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz )
|
||||
static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz, MPI_Comm comm )
|
||||
{
|
||||
if ( N_blobs==0 )
|
||||
return 0;
|
||||
@@ -210,7 +210,7 @@ static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int
|
||||
}
|
||||
}
|
||||
ASSERT(max_id<N_blobs);
|
||||
MPI_Allreduce(local_size,global_size,N_blobs,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(local_size,global_size,N_blobs,MPI_DOUBLE,MPI_SUM,comm);
|
||||
std::vector<std::pair<double,int> > map1(N_blobs);
|
||||
int N_blobs2 = 0;
|
||||
for (int i=0; i<N_blobs; i++) {
|
||||
@@ -233,13 +233,13 @@ static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int
|
||||
PROFILE_STOP("ReorderBlobIDs2",1);
|
||||
return N_blobs2;
|
||||
}
|
||||
void ReorderBlobIDs( BlobIDArray& ID )
|
||||
void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm )
|
||||
{
|
||||
PROFILE_START("ReorderBlobIDs");
|
||||
int tmp = ID.max()+1;
|
||||
int N_blobs = 0;
|
||||
MPI_Allreduce(&tmp,&N_blobs,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD);
|
||||
ReorderBlobIDs2(ID,N_blobs,1,1,1);
|
||||
MPI_Allreduce(&tmp,&N_blobs,1,MPI_INT,MPI_MAX,comm);
|
||||
ReorderBlobIDs2(ID,N_blobs,1,1,1,comm);
|
||||
PROFILE_STOP("ReorderBlobIDs");
|
||||
}
|
||||
|
||||
@@ -257,7 +257,8 @@ static void updateRemoteIds(
|
||||
const std::vector<int>& neighbors,
|
||||
int N_send, const std::vector<int>& N_recv,
|
||||
int64_t *send_buf, std::vector<int64_t*>& recv_buf,
|
||||
std::map<int64_t,int64_t>& remote_map )
|
||||
std::map<int64_t,int64_t>& remote_map,
|
||||
MPI_Comm comm )
|
||||
{
|
||||
std::vector<MPI_Request> send_req(neighbors.size());
|
||||
std::vector<MPI_Request> recv_req(neighbors.size());
|
||||
@@ -269,8 +270,8 @@ static void updateRemoteIds(
|
||||
send_buf[2*i+1] = it->second.new_id;
|
||||
}
|
||||
for (size_t i=0; i<neighbors.size(); i++) {
|
||||
MPI_Isend( send_buf, 2*N_send, MPI_LONG_LONG, neighbors[i], 0, MPI_COMM_WORLD, &send_req[i] );
|
||||
MPI_Irecv( recv_buf[i], 2*N_recv[i], MPI_LONG_LONG, neighbors[i], 0, MPI_COMM_WORLD, &recv_req[i] );
|
||||
MPI_Isend( send_buf, 2*N_send, MPI_LONG_LONG, neighbors[i], 0, comm, &send_req[i] );
|
||||
MPI_Irecv( recv_buf[i], 2*N_recv[i], MPI_LONG_LONG, neighbors[i], 0, comm, &recv_req[i] );
|
||||
}
|
||||
for (it=map.begin(); it!=map.end(); ++it) {
|
||||
remote_map[it->first] = it->second.new_id;
|
||||
@@ -301,19 +302,18 @@ static bool updateLocalIds( const std::map<int64_t,int64_t>& remote_map,
|
||||
return changed;
|
||||
}
|
||||
static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
|
||||
int nblobs, BlobIDArray& IDs )
|
||||
int nblobs, BlobIDArray& IDs, MPI_Comm comm )
|
||||
{
|
||||
PROFILE_START("LocalToGlobalIDs",1);
|
||||
const int rank = rank_info.rank[1][1][1];
|
||||
int nprocs;
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
int nprocs = comm_size(comm);
|
||||
const int ngx = (IDs.size(0)-nx)/2;
|
||||
const int ngy = (IDs.size(1)-ny)/2;
|
||||
const int ngz = (IDs.size(2)-nz)/2;
|
||||
// Get the number of blobs for each rank
|
||||
std::vector<int> N_blobs(nprocs,0);
|
||||
PROFILE_START("LocalToGlobalIDs-Allgather",1);
|
||||
MPI_Allgather(&nblobs,1,MPI_INT,getPtr(N_blobs),1,MPI_INT,MPI_COMM_WORLD);
|
||||
MPI_Allgather(&nblobs,1,MPI_INT,getPtr(N_blobs),1,MPI_INT,comm);
|
||||
PROFILE_STOP("LocalToGlobalIDs-Allgather",1);
|
||||
int64_t N_blobs_tot = 0;
|
||||
int offset = 0;
|
||||
@@ -329,7 +329,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
|
||||
}
|
||||
const BlobIDArray LocalIDs = IDs;
|
||||
// Copy the ids and get the neighbors through the halos
|
||||
fillHalo<BlobIDType> fillData(rank_info,nx,ny,nz,1,1,1,0,1,true,true,true);
|
||||
fillHalo<BlobIDType> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1,true,true,true);
|
||||
fillData.fill(IDs);
|
||||
// Create a list of all neighbor ranks (excluding self)
|
||||
std::vector<int> neighbors;
|
||||
@@ -364,8 +364,8 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
|
||||
std::vector<MPI_Request> recv_req(neighbors.size());
|
||||
std::vector<MPI_Status> status(neighbors.size());
|
||||
for (size_t i=0; i<neighbors.size(); i++) {
|
||||
MPI_Isend( &N_send, 1, MPI_INT, neighbors[i], 0, MPI_COMM_WORLD, &send_req[i] );
|
||||
MPI_Irecv( &N_recv[i], 1, MPI_INT, neighbors[i], 0, MPI_COMM_WORLD, &recv_req[i] );
|
||||
MPI_Isend( &N_send, 1, MPI_INT, neighbors[i], 0, comm, &send_req[i] );
|
||||
MPI_Irecv( &N_recv[i], 1, MPI_INT, neighbors[i], 0, comm, &recv_req[i] );
|
||||
}
|
||||
MPI_Waitall(neighbors.size(),getPtr(send_req),getPtr(status));
|
||||
MPI_Waitall(neighbors.size(),getPtr(recv_req),getPtr(status));
|
||||
@@ -392,13 +392,13 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
|
||||
while ( 1 ) {
|
||||
iteration++;
|
||||
// Send the local ids and their new value to all neighbors
|
||||
updateRemoteIds( map, neighbors, N_send, N_recv,send_buf, recv_buf, remote_map );
|
||||
updateRemoteIds( map, neighbors, N_send, N_recv,send_buf, recv_buf, remote_map, comm );
|
||||
// Compute a new local id for each local id
|
||||
bool changed = updateLocalIds( remote_map, map );
|
||||
// Check if we are finished
|
||||
int test = changed ? 1:0;
|
||||
int result = 0;
|
||||
MPI_Allreduce(&test,&result,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&test,&result,1,MPI_INT,MPI_SUM,comm);
|
||||
if ( result==0 )
|
||||
break;
|
||||
}
|
||||
@@ -421,10 +421,10 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
|
||||
}
|
||||
}
|
||||
// Fill the ghosts
|
||||
fillHalo<BlobIDType> fillData2(rank_info,nx,ny,nz,1,1,1,0,1,true,true,true);
|
||||
fillHalo<BlobIDType> fillData2(comm,rank_info,nx,ny,nz,1,1,1,0,1,true,true,true);
|
||||
fillData2.fill(IDs);
|
||||
// Reorder based on size (and compress the id space
|
||||
int N_blobs_global = ReorderBlobIDs2(IDs,N_blobs_tot,ngx,ngy,ngz);
|
||||
int N_blobs_global = ReorderBlobIDs2(IDs,N_blobs_tot,ngx,ngy,ngz,comm);
|
||||
// Finished
|
||||
delete [] send_buf;
|
||||
for (size_t i=0; i<neighbors.size(); i++)
|
||||
@@ -434,24 +434,24 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
|
||||
}
|
||||
int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
|
||||
const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS,
|
||||
BlobIDArray& GlobalBlobID )
|
||||
BlobIDArray& GlobalBlobID, MPI_Comm comm )
|
||||
{
|
||||
PROFILE_START("ComputeGlobalBlobIDs");
|
||||
// First compute the local ids
|
||||
int nblobs = ComputeLocalBlobIDs(Phase,SignDist,vF,vS,GlobalBlobID,false);
|
||||
// Compute the global ids
|
||||
int nglobal = LocalToGlobalIDs( nx, ny, nz, rank_info, nblobs, GlobalBlobID );
|
||||
int nglobal = LocalToGlobalIDs( nx, ny, nz, rank_info, nblobs, GlobalBlobID, comm );
|
||||
PROFILE_STOP("ComputeGlobalBlobIDs");
|
||||
return nglobal;
|
||||
}
|
||||
int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
|
||||
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID )
|
||||
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm )
|
||||
{
|
||||
PROFILE_START("ComputeGlobalPhaseComponent");
|
||||
// First compute the local ids
|
||||
int nblobs = ComputeLocalPhaseComponent(PhaseID,VALUE,GlobalBlobID,false);
|
||||
// Compute the global ids
|
||||
int nglobal = LocalToGlobalIDs( nx, ny, nz, rank_info, nblobs, GlobalBlobID );
|
||||
int nglobal = LocalToGlobalIDs( nx, ny, nz, rank_info, nblobs, GlobalBlobID, comm );
|
||||
PROFILE_STOP("ComputeGlobalPhaseComponent");
|
||||
return nglobal;
|
||||
}
|
||||
@@ -469,29 +469,26 @@ template<> inline MPI_Datatype getMPIType<int64_t>() {
|
||||
return MPI_DOUBLE;
|
||||
}
|
||||
template<class TYPE>
|
||||
void gatherSet( std::set<TYPE>& set )
|
||||
void gatherSet( std::set<TYPE>& set, MPI_Comm comm )
|
||||
{
|
||||
int nprocs;
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
int nprocs = comm_size(comm);
|
||||
MPI_Datatype type = getMPIType<TYPE>();
|
||||
std::vector<TYPE> send_data(set.begin(),set.end());
|
||||
int send_count = send_data.size();
|
||||
std::vector<int> recv_count(nprocs,0), recv_disp(nprocs,0);
|
||||
MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,MPI_COMM_WORLD);
|
||||
MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm);
|
||||
for (int i=1; i<nprocs; i++)
|
||||
recv_disp[i] = recv_disp[i-1] + recv_count[i-1];
|
||||
std::vector<TYPE> recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]);
|
||||
MPI_Allgatherv(getPtr(send_data),send_count,type,
|
||||
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,
|
||||
MPI_COMM_WORLD);
|
||||
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm);
|
||||
for (size_t i=0; i<recv_data.size(); i++)
|
||||
set.insert(recv_data[i]);
|
||||
}
|
||||
template<class TYPE>
|
||||
void gatherSrcIDMap( std::map<TYPE,std::set<TYPE> >& src_map )
|
||||
void gatherSrcIDMap( std::map<TYPE,std::set<TYPE> >& src_map, MPI_Comm comm )
|
||||
{
|
||||
int nprocs;
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
int nprocs = comm_size(comm);
|
||||
MPI_Datatype type = getMPIType<TYPE>();
|
||||
std::vector<TYPE> send_data;
|
||||
typename std::map<TYPE,std::set<TYPE> >::const_iterator it;
|
||||
@@ -506,13 +503,12 @@ void gatherSrcIDMap( std::map<TYPE,std::set<TYPE> >& src_map )
|
||||
}
|
||||
int send_count = send_data.size();
|
||||
std::vector<int> recv_count(nprocs,0), recv_disp(nprocs,0);
|
||||
MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,MPI_COMM_WORLD);
|
||||
MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm);
|
||||
for (int i=1; i<nprocs; i++)
|
||||
recv_disp[i] = recv_disp[i-1] + recv_count[i-1];
|
||||
std::vector<TYPE> recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]);
|
||||
MPI_Allgatherv(getPtr(send_data),send_count,type,
|
||||
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,
|
||||
MPI_COMM_WORLD);
|
||||
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm);
|
||||
size_t i=0;
|
||||
while ( i < recv_data.size() ) {
|
||||
int id = recv_data[i];
|
||||
@@ -533,7 +529,7 @@ void addSrcDstIDs( BlobIDType src_id, std::map<BlobIDType,std::set<BlobIDType> >
|
||||
addSrcDstIDs(*it,dst_map,src_map,dst,src);
|
||||
}
|
||||
}
|
||||
ID_map_struct computeIDMap( const BlobIDArray& ID1, const BlobIDArray& ID2 )
|
||||
ID_map_struct computeIDMap( const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm )
|
||||
{
|
||||
ASSERT(ID1.size()==ID2.size());
|
||||
PROFILE_START("computeIDMap");
|
||||
@@ -552,9 +548,9 @@ ID_map_struct computeIDMap( const BlobIDArray& ID1, const BlobIDArray& ID2 )
|
||||
}
|
||||
}
|
||||
// Communicate the src/dst ids and src id map to all processors and reduce
|
||||
gatherSet( src_set );
|
||||
gatherSet( dst_set );
|
||||
gatherSrcIDMap( src_map );
|
||||
gatherSet( src_set, comm );
|
||||
gatherSet( dst_set, comm );
|
||||
gatherSrcIDMap( src_map, comm );
|
||||
// Compute the dst id map
|
||||
std::map<BlobIDType,std::set<BlobIDType> > dst_map; // Map of the dst ids for each src id
|
||||
for (std::map<BlobIDType,std::set<BlobIDType> >::const_iterator it=src_map.begin(); it!=src_map.end(); ++it) {
|
||||
@@ -703,8 +699,7 @@ void renumberIDs( const std::vector<BlobIDType>& new_ids, BlobIDArray& IDs )
|
||||
******************************************************************/
|
||||
void writeIDMap( const ID_map_struct& map, long long int timestep, const std::string& filename )
|
||||
{
|
||||
int rank;
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
int rank = MPI_WORLD_RANK();
|
||||
if ( rank!=0 )
|
||||
return;
|
||||
bool empty = map.created.empty() && map.destroyed.empty() &&
|
||||
|
||||
@@ -58,7 +58,7 @@ int ComputeLocalPhaseComponent( const IntArray &PhaseID, int &VALUE, IntArray &C
|
||||
*/
|
||||
int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
|
||||
const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS,
|
||||
BlobIDArray& GlobalBlobID );
|
||||
BlobIDArray& GlobalBlobID, MPI_Comm comm );
|
||||
|
||||
|
||||
/*!
|
||||
@@ -75,7 +75,7 @@ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_inf
|
||||
* @return Return the number of components in the specified phase
|
||||
*/
|
||||
int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
|
||||
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID );
|
||||
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm );
|
||||
|
||||
|
||||
/*!
|
||||
@@ -87,7 +87,7 @@ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& r
|
||||
* @param[in] nz Number of elements in the z-direction
|
||||
* @param[in/out] ID The ids of the blobs
|
||||
*/
|
||||
void ReorderBlobIDs( BlobIDArray& ID );
|
||||
void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm );
|
||||
|
||||
|
||||
typedef std::pair<BlobIDType,std::vector<BlobIDType> > BlobIDSplitStruct;
|
||||
@@ -118,7 +118,7 @@ struct ID_map_struct {
|
||||
* @param[in] ID1 The blob ids at the first timestep
|
||||
* @param[in] ID2 The blob ids at the second timestep
|
||||
*/
|
||||
ID_map_struct computeIDMap( const BlobIDArray& ID1, const BlobIDArray& ID2 );
|
||||
ID_map_struct computeIDMap( const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm );
|
||||
|
||||
|
||||
/*!
|
||||
|
||||
@@ -51,7 +51,7 @@ public:
|
||||
* @param[in] tag Initial tag to use for the communication (we will require tag:tag+26)
|
||||
* @param[in] depth Maximum depth to support
|
||||
*/
|
||||
fillHalo( const RankInfoStruct& info, int nx, int ny, int nz,
|
||||
fillHalo( MPI_Comm comm, const RankInfoStruct& info, int nx, int ny, int nz,
|
||||
int ngx, int ngy, int ngz, int tag, int depth,
|
||||
bool fill_face=true, bool fill_edge=true, bool fill_corner=true );
|
||||
|
||||
|
||||
@@ -10,12 +10,12 @@
|
||||
* Structure to store the rank info *
|
||||
********************************************************/
|
||||
template<class TYPE>
|
||||
fillHalo<TYPE>::fillHalo( const RankInfoStruct& info0, int nx0, int ny0, int nz0,
|
||||
fillHalo<TYPE>::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0, int ny0, int nz0,
|
||||
int ngx0, int ngy0, int ngz0, int tag0, int depth0,
|
||||
bool fill_face, bool fill_edge, bool fill_corner ):
|
||||
info(info0), nx(nx0), ny(ny0), nz(nz0), ngx(ngx0), ngy(ngy0), ngz(ngz0), depth(depth0)
|
||||
{
|
||||
comm = MPI_COMM_WORLD;
|
||||
comm = comm0;
|
||||
datatype = getMPItype<TYPE>();
|
||||
// Set the fill pattern
|
||||
memset(fill_pattern,0,sizeof(fill_pattern));
|
||||
|
||||
@@ -171,7 +171,7 @@ inline void SSO(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){
|
||||
xdim=Dm.Nx-2;
|
||||
ydim=Dm.Ny-2;
|
||||
zdim=Dm.Nz-2;
|
||||
fillHalo<double> fillData(Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
|
||||
fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
|
||||
|
||||
int count = 0;
|
||||
while (count < timesteps){
|
||||
|
||||
@@ -140,6 +140,11 @@ int MPI_Init(int*,char***)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
int MPI_Init_thread(int*,char***, int required, int *provided )
|
||||
{
|
||||
*provided = required;
|
||||
return 0;
|
||||
}
|
||||
int MPI_Finalize()
|
||||
{
|
||||
return 0;
|
||||
@@ -223,6 +228,12 @@ int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
|
||||
ERROR("Not implimented yet");
|
||||
return 0;
|
||||
}
|
||||
int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
|
||||
MPI_Op op, int root, MPI_Comm comm)
|
||||
{
|
||||
ERROR("Not implimented yet");
|
||||
return 0;
|
||||
}
|
||||
int MPI_Comm_group(MPI_Comm comm, MPI_Group *group)
|
||||
{
|
||||
ERROR("Not implimented yet");
|
||||
@@ -233,10 +244,23 @@ int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm)
|
||||
ERROR("Not implimented yet");
|
||||
return 0;
|
||||
}
|
||||
int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
|
||||
{
|
||||
*newcomm = comm;
|
||||
return 0;
|
||||
}
|
||||
double MPI_Wtime( void )
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
int MPI_Comm_free(MPI_Comm *group)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
int MPI_Group_free(MPI_Group *group)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
@@ -17,13 +17,19 @@
|
||||
typedef int MPI_Status;
|
||||
#define MPI_COMM_WORLD 0
|
||||
#define MPI_COMM_SELF 0
|
||||
#define MPI_COMM_NULL -1
|
||||
#define MPI_STATUS_IGNORE NULL
|
||||
enum MPI_Datatype { MPI_LOGICAL, MPI_CHAR, MPI_UNSIGNED_CHAR, MPI_INT,
|
||||
MPI_UNSIGNED, MPI_LONG, MPI_UNSIGNED_LONG, MPI_LONG_LONG, MPI_FLOAT, MPI_DOUBLE };
|
||||
enum MPI_Op { MPI_MIN, MPI_MAX, MPI_SUM };
|
||||
enum MPI_Group { };
|
||||
typedef int MPI_Group;
|
||||
#define MPI_THREAD_SINGLE 0
|
||||
#define MPI_THREAD_FUNNELED 1
|
||||
#define MPI_THREAD_SERIALIZED 2
|
||||
#define MPI_THREAD_MULTIPLE 3
|
||||
// Fake MPI functions
|
||||
int MPI_Init(int*,char***);
|
||||
int MPI_Init_thread( int *argc, char ***argv, int required, int *provided );
|
||||
int MPI_Finalize();
|
||||
int MPI_Comm_size( MPI_Comm, int *size );
|
||||
int MPI_Comm_rank( MPI_Comm, int *rank );
|
||||
@@ -52,24 +58,43 @@
|
||||
void *recvbuf, int recvcount, MPI_Datatype recvtype,
|
||||
int source, int recvtag,
|
||||
MPI_Comm comm, MPI_Status *status);
|
||||
int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
|
||||
MPI_Op op, int root, MPI_Comm comm);
|
||||
double MPI_Wtime( void );
|
||||
int MPI_Comm_group(MPI_Comm comm, MPI_Group *group);
|
||||
int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm);
|
||||
int MPI_Comm_free(MPI_Comm *group);
|
||||
int MPI_Group_free(MPI_Group *group);
|
||||
int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
|
||||
#endif
|
||||
|
||||
|
||||
//! Get the size of the MPI_Comm
|
||||
// Note: this is a thread and interrupt safe function
|
||||
inline int comm_size( MPI_Comm comm ) {
|
||||
int size = 1;
|
||||
MPI_Comm_size( comm, &size );
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
//! Get the rank of the MPI_Comm
|
||||
// Note: this is a thread and interrupt safe function
|
||||
inline int comm_rank( MPI_Comm comm ) {
|
||||
int rank = 1;
|
||||
MPI_Comm_rank( comm, &rank );
|
||||
return rank;
|
||||
}
|
||||
|
||||
|
||||
//! Get the size of MPI_COMM_WORLD
|
||||
inline int MPI_WORLD_SIZE( ) {
|
||||
int size = 1;
|
||||
MPI_Comm_size( MPI_COMM_WORLD, &size );
|
||||
return size;
|
||||
return comm_size( MPI_COMM_WORLD );
|
||||
}
|
||||
|
||||
//! Get the size of MPI_COMM_WORLD
|
||||
inline int MPI_WORLD_RANK( ) {
|
||||
int rank = 0;
|
||||
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
|
||||
return rank;
|
||||
return comm_rank( MPI_COMM_WORLD );
|
||||
}
|
||||
|
||||
//! Return the appropriate MPI datatype for a class
|
||||
|
||||
@@ -505,7 +505,7 @@ void TwoPhase::AssignComponentLabels()
|
||||
|
||||
// Fewer non-wetting phase features are present
|
||||
//NumberComponents_NWP = ComputeGlobalPhaseComponent(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.rank_info,PhaseID,LabelNWP,Label_NWP);
|
||||
NumberComponents_NWP = ComputeGlobalBlobIDs(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.rank_info,SDs,SDn,solid_isovalue,fluid_isovalue,Label_NWP);
|
||||
NumberComponents_NWP = ComputeGlobalBlobIDs(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.rank_info,SDs,SDn,solid_isovalue,fluid_isovalue,Label_NWP,Dm.Comm);
|
||||
}
|
||||
|
||||
void TwoPhase::ComponentAverages()
|
||||
@@ -761,8 +761,8 @@ void TwoPhase::ComponentAverages()
|
||||
RecvBuffer.resize(BLOB_AVG_COUNT,NumberComponents_NWP);
|
||||
|
||||
/* for (int b=0; b<NumberComponents_NWP; b++){
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(Dm.Comm);
|
||||
MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm.Comm);
|
||||
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_NWP(idx,b)=RecvBuffer(idx);
|
||||
}
|
||||
*/
|
||||
@@ -865,7 +865,7 @@ void TwoPhase::ComponentAverages()
|
||||
// reduce the wetting phase averages
|
||||
for (int b=0; b<NumberComponents_WP; b++){
|
||||
MPI_Barrier(Dm.Comm);
|
||||
// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.get(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.get(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm.Comm);
|
||||
MPI_Reduce(&ComponentAverages_WP(0,b),RecvBuffer.get(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm.Comm);
|
||||
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_WP(idx,b)=RecvBuffer(idx);
|
||||
}
|
||||
@@ -1063,7 +1063,7 @@ void TwoPhase::WriteSurfaces(int logcount)
|
||||
meshData[1].mesh = ws_mesh;
|
||||
meshData[2].meshName = "ns-tris";
|
||||
meshData[2].mesh = ns_mesh;
|
||||
IO::writeData( logcount, meshData, 2);
|
||||
IO::writeData( logcount, meshData, 2, Dm.Comm );
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -42,8 +42,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -114,31 +115,31 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
// **************************************************************
|
||||
|
||||
@@ -168,7 +169,7 @@ int main(int argc, char **argv)
|
||||
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -450,7 +451,7 @@ int main(int argc, char **argv)
|
||||
PM.close();
|
||||
// printf("File porosity = %f\n", double(sum)/N);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//...........................................................................
|
||||
// Write the communcation structure into a file for debugging
|
||||
@@ -587,7 +588,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -598,46 +599,46 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
//**********************************************************************************
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
@@ -668,48 +669,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//......................................................................................
|
||||
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
|
||||
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
|
||||
@@ -846,41 +847,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -913,7 +914,7 @@ int main(int argc, char **argv)
|
||||
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
|
||||
*/ //......................................................................................
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -993,42 +994,42 @@ int main(int argc, char **argv)
|
||||
PackValues(sendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
@@ -1063,7 +1064,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
|
||||
@@ -1157,42 +1158,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1292,18 +1293,18 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Send all the D3Q7 distributions
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of D3Q7 communication
|
||||
@@ -1344,42 +1345,42 @@ int main(int argc, char **argv)
|
||||
PackValues(sendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
@@ -1408,7 +1409,7 @@ int main(int argc, char **argv)
|
||||
UnpackValues(recvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Iteration completed!
|
||||
timestep++;
|
||||
//...................................................................
|
||||
@@ -1458,7 +1459,7 @@ int main(int argc, char **argv)
|
||||
fwrite(ColorGrad,8,3*N,COLORGRAD);
|
||||
fclose(COLORGRAD);
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -178,8 +178,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -282,38 +283,38 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&xIntPos,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&xIntPos,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
// **************************************************************
|
||||
double Ps = -(das-dbs)/(das+dbs);
|
||||
@@ -345,7 +346,7 @@ int main(int argc, char **argv)
|
||||
printf("********************************************************\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -684,14 +685,14 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the sphere packing \n");
|
||||
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//.......................................................................
|
||||
// sprintf(LocalRankString,"%05d",rank);
|
||||
@@ -724,7 +725,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
sum_local = 1.0*sum;
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = porosity*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
|
||||
@@ -858,7 +859,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -869,88 +870,88 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
*/ //**********************************************************************************
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
@@ -981,48 +982,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//......................................................................................
|
||||
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
|
||||
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
|
||||
@@ -1137,7 +1138,7 @@ int main(int argc, char **argv)
|
||||
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
|
||||
dvc_AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
|
||||
//......................................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
|
||||
dvc_CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
|
||||
dvc_CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
|
||||
@@ -1245,41 +1246,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -1347,7 +1348,7 @@ int main(int argc, char **argv)
|
||||
recvMeshData_YZ = new double [recvCount_YZ];
|
||||
recvMeshData_XZ = new double [recvCount_XZ];
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1527,7 +1528,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
//...........................................................................
|
||||
@@ -1599,7 +1600,7 @@ int main(int argc, char **argv)
|
||||
dvc_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
|
||||
dvc_CopyToDevice(Den,cDen,2*N*sizeof(double));
|
||||
dvc_Barrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
// Pack the buffers (zeros out the halo region)
|
||||
dvc_PackDenD3Q7(dvcRecvList_x,recvCount_x,recvbuf_x,2,Den,N);
|
||||
@@ -1638,42 +1639,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1725,7 +1726,7 @@ int main(int argc, char **argv)
|
||||
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double));
|
||||
dvc_CopyToHost(Press.data,Pressure,N*sizeof(double));
|
||||
dvc_CopyToHost(Vel,Velocity,3*N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//...........................................................................
|
||||
|
||||
timestep=0;
|
||||
@@ -1821,42 +1822,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1957,18 +1958,18 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the D3Q7 distributions
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of D3Q7 communication
|
||||
@@ -2012,42 +2013,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -2083,7 +2084,7 @@ int main(int argc, char **argv)
|
||||
dvc_UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
|
||||
dvc_UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Iteration completed!
|
||||
timestep++;
|
||||
@@ -2122,7 +2123,7 @@ int main(int argc, char **argv)
|
||||
// Fill in the halo region for the mesh gradients and curvature
|
||||
//...........................................................................
|
||||
// Pressure
|
||||
CommunicateMeshHalo(Press, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Press, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2146,7 +2147,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Mean Curvature
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(MeanCurvature, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(MeanCurvature, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2171,7 +2172,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Gaussian Curvature
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(GaussCurvature, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(GaussCurvature, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2195,7 +2196,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Gradient of the phase indicator field
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_x, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_x, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2217,7 +2218,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_y, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_y, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2239,7 +2240,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_z, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_z, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2363,27 +2364,27 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
}
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
// Phase averages
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Barrier(comm);
|
||||
//.........................................................................
|
||||
// Compute the change in the total surface energy based on the defined interval
|
||||
// See McClure, Prins and Miller (2013)
|
||||
@@ -2450,7 +2451,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//************************************************************************/
|
||||
dvc_Barrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
if (rank==0) printf("-------------------------------------------------------------------\n");
|
||||
// Compute the walltime per timestep
|
||||
@@ -2488,7 +2489,7 @@ int main(int argc, char **argv)
|
||||
*/ //************************************************************************/
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -70,8 +70,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -135,24 +136,24 @@ int main(int argc, char **argv)
|
||||
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
|
||||
double rlx_setA = 1.f/tau;
|
||||
@@ -175,7 +176,7 @@ int main(int argc, char **argv)
|
||||
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -456,7 +457,7 @@ int main(int argc, char **argv)
|
||||
PM.close();
|
||||
// printf("File porosity = %f\n", double(sum)/N);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//...........................................................................
|
||||
// Write the communcation structure into a file for debugging
|
||||
@@ -593,7 +594,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -604,46 +605,46 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
//**********************************************************************************
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
@@ -674,48 +675,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//......................................................................................
|
||||
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
|
||||
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
|
||||
@@ -915,41 +916,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -982,7 +983,7 @@ int main(int argc, char **argv)
|
||||
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
|
||||
//......................................................................................
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1022,7 +1023,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
// Old cuda timer is below
|
||||
// cudaEvent_t start, stop;
|
||||
@@ -1135,42 +1136,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//...................................................................................
|
||||
@@ -1259,7 +1260,7 @@ int main(int argc, char **argv)
|
||||
//*****************************************************************************
|
||||
//*****************************************************************************
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Iteration completed!
|
||||
timestep++;
|
||||
//...................................................................
|
||||
@@ -1268,7 +1269,7 @@ int main(int argc, char **argv)
|
||||
|
||||
// cudaThreadSynchronize();
|
||||
dvc_Barrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
|
||||
cputime = stoptime - starttime;
|
||||
@@ -1303,7 +1304,7 @@ int main(int argc, char **argv)
|
||||
// dvc_CopyToDevice(velocity, vel, 3*dist_mem_size, dvc_CopyToDeviceDeviceToHost);
|
||||
//..............................................................................
|
||||
// cudaThreadSynchronize();
|
||||
// MPI_Barrier(MPI_COMM_WORLD);
|
||||
// MPI_Barrier(comm);
|
||||
//............................................................
|
||||
//....Write the z-velocity to test poiseuille flow............
|
||||
// double vz,vz_avg;
|
||||
@@ -1332,7 +1333,7 @@ int main(int argc, char **argv)
|
||||
// free (velocity); free(id);
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -559,8 +559,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -624,24 +625,24 @@ int main(int argc, char **argv)
|
||||
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&iterMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&iterMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
|
||||
double rlx_setA = 1.f/tau;
|
||||
@@ -664,7 +665,7 @@ int main(int argc, char **argv)
|
||||
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -945,7 +946,7 @@ int main(int argc, char **argv)
|
||||
PM.close();
|
||||
// printf("File porosity = %f\n", double(sum)/N);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//...........................................................................
|
||||
// Write the communcation structure into a file for debugging
|
||||
@@ -1082,7 +1083,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -1093,46 +1094,46 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
//**********************************************************************************
|
||||
//recvCount_x = sendCount_x;
|
||||
//recvCount_X = sendCount_X;
|
||||
@@ -1156,7 +1157,7 @@ int main(int argc, char **argv)
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the appropriate values
|
||||
// int tag = 5;
|
||||
// MPI_Sendrecv(sendCount_x,1,MPI_INT,rank_x,tag,sendCount_X,1,MPI_INT,MPI_COMM_WORLD,req);
|
||||
// MPI_Sendrecv(sendCount_x,1,MPI_INT,rank_x,tag,sendCount_X,1,MPI_INT,comm,req);
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
|
||||
@@ -1186,48 +1187,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//......................................................................................
|
||||
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
|
||||
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
|
||||
@@ -1427,41 +1428,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -1494,7 +1495,7 @@ int main(int argc, char **argv)
|
||||
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
|
||||
//......................................................................................
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1534,7 +1535,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
// Old cuda timer is below
|
||||
// cudaEvent_t start, stop;
|
||||
@@ -1632,42 +1633,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//...................................................................................
|
||||
@@ -1757,7 +1758,7 @@ int main(int argc, char **argv)
|
||||
//*****************************************************************************
|
||||
//*****************************************************************************
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Iteration completed!
|
||||
iter++;
|
||||
//...................................................................
|
||||
@@ -1765,7 +1766,7 @@ int main(int argc, char **argv)
|
||||
//************************************************************************/
|
||||
|
||||
cudaThreadSynchronize();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
|
||||
cputime = stoptime - starttime;
|
||||
@@ -1801,7 +1802,7 @@ int main(int argc, char **argv)
|
||||
cudaMemcpy(velocity, vel, 3*dist_mem_size, cudaMemcpyDeviceToHost);
|
||||
//..............................................................................
|
||||
cudaThreadSynchronize();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//............................................................
|
||||
//....Write the z-velocity to test poiseuille flow............
|
||||
double vz,vz_avg;
|
||||
@@ -1830,7 +1831,7 @@ int main(int argc, char **argv)
|
||||
free (velocity); free(id);
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -66,9 +66,11 @@ int main(int argc, char *argv[])
|
||||
#ifdef useMPI
|
||||
MPI_Status stat;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_size(comm,&numprocs);
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
#else
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
numprocs = 1;
|
||||
rank = 0;
|
||||
#endif
|
||||
@@ -124,27 +126,27 @@ int main(int argc, char *argv[])
|
||||
#ifdef useMPI
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
#endif
|
||||
|
||||
@@ -242,9 +244,9 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
}
|
||||
#ifdef useMPI //............................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Bcast(&id[0],N,MPI_CHAR,0,MPI_COMM_WORLD);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Bcast(&id[0],N,MPI_CHAR,0,comm);
|
||||
MPI_Barrier(comm);
|
||||
#endif
|
||||
if (rank == 0) printf("Domain set.\n");
|
||||
//...........................................................................
|
||||
|
||||
@@ -104,8 +104,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -176,31 +177,31 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
// **************************************************************
|
||||
|
||||
@@ -230,7 +231,7 @@ int main(int argc, char **argv)
|
||||
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -512,7 +513,7 @@ int main(int argc, char **argv)
|
||||
PM.close();
|
||||
// printf("File porosity = %f\n", double(sum)/N);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//...........................................................................
|
||||
// Write the communcation structure into a file for debugging
|
||||
@@ -649,7 +650,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -660,46 +661,46 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
//**********************************************************************************
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
@@ -730,48 +731,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//......................................................................................
|
||||
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
|
||||
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
|
||||
@@ -993,41 +994,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -1060,7 +1061,7 @@ int main(int argc, char **argv)
|
||||
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
|
||||
*/ //......................................................................................
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1125,42 +1126,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1206,7 +1207,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
// Old cuda timer is below
|
||||
// cudaEvent_t start, stop;
|
||||
@@ -1308,42 +1309,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1445,18 +1446,18 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the D3Q7 distributions
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of D3Q7 communication
|
||||
@@ -1506,42 +1507,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1576,7 +1577,7 @@ int main(int argc, char **argv)
|
||||
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
|
||||
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Iteration completed!
|
||||
timestep++;
|
||||
|
||||
@@ -1586,7 +1587,7 @@ int main(int argc, char **argv)
|
||||
|
||||
// cudaThreadSynchronize();
|
||||
dvc_Barrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
|
||||
cputime = stoptime - starttime;
|
||||
@@ -1615,7 +1616,7 @@ int main(int argc, char **argv)
|
||||
//************************************************************************/
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -107,8 +107,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -202,35 +203,35 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
// **************************************************************
|
||||
double Ps = -(das-dbs)/(das+dbs);
|
||||
@@ -262,7 +263,7 @@ int main(int argc, char **argv)
|
||||
printf("********************************************************\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -560,14 +561,14 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the sphere packing \n");
|
||||
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//.......................................................................
|
||||
// sprintf(LocalRankString,"%05d",rank);
|
||||
@@ -717,7 +718,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -728,88 +729,88 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
*/ //**********************************************************************************
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
@@ -840,48 +841,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//......................................................................................
|
||||
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
|
||||
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
|
||||
@@ -1103,41 +1104,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -1170,7 +1171,7 @@ int main(int argc, char **argv)
|
||||
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
|
||||
*/ //......................................................................................
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1321,42 +1322,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1402,7 +1403,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
|
||||
@@ -1493,42 +1494,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1629,18 +1630,18 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the D3Q7 distributions
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of D3Q7 communication
|
||||
@@ -1684,42 +1685,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1754,7 +1755,7 @@ int main(int argc, char **argv)
|
||||
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
|
||||
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Iteration completed!
|
||||
timestep++;
|
||||
//...................................................................
|
||||
@@ -1765,7 +1766,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
dvc_Barrier();
|
||||
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//...........................................................................
|
||||
// Compute areas using porous medium marching cubes algorithm
|
||||
// McClure, Adalsteinsson, et al. (2007)
|
||||
@@ -1934,15 +1935,15 @@ int main(int argc, char **argv)
|
||||
//*******************************************************************
|
||||
}
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.........................................................................
|
||||
// Compute the change in the total surface energy based on the defined interval
|
||||
// See McClure, Prins and Miller (2013)
|
||||
@@ -1971,7 +1972,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//************************************************************************/
|
||||
dvc_Barrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
if (rank==0) printf("-------------------------------------------------------------------\n");
|
||||
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
|
||||
@@ -2008,7 +2009,7 @@ int main(int argc, char **argv)
|
||||
*/ //************************************************************************/
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -98,8 +98,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -194,40 +195,40 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&SimCount,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&SimDelta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&SimCount,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&SimDelta,1,MPI_DOUBLE,0,comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//int RESTART_INTERVAL=interval;
|
||||
|
||||
@@ -279,7 +280,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -357,14 +358,14 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the sphere packing \n");
|
||||
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
if (rank == 0){
|
||||
// Compute the Sauter mean diameter
|
||||
@@ -378,7 +379,7 @@ int main(int argc, char **argv)
|
||||
D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx;
|
||||
printf("Sauter Mean Diameter (computed from sphere packing) = %f \n ",D);
|
||||
}
|
||||
MPI_Bcast(&D,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&D,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
//.......................................................................
|
||||
// sprintf(LocalRankString,"%05d",rank);
|
||||
@@ -416,7 +417,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
sum_local = 1.0*sum;
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = porosity*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
|
||||
@@ -432,7 +433,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
|
||||
|
||||
/* // Generate the residual NWP
|
||||
@@ -597,7 +598,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -608,7 +609,7 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
CommunicateSendRecvCounts( MPI_COMM_WORLD, sendtag, recvtag,
|
||||
CommunicateSendRecvCounts( comm, sendtag, recvtag,
|
||||
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy,
|
||||
rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
@@ -651,7 +652,7 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
CommunicateRecvLists( MPI_COMM_WORLD, sendtag, recvtag,
|
||||
CommunicateRecvLists( comm, sendtag, recvtag,
|
||||
sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z,
|
||||
sendList_xy, sendList_XY, sendList_xY, sendList_Xy,
|
||||
sendList_xz, sendList_XZ, sendList_xZ, sendList_Xz,
|
||||
@@ -784,7 +785,7 @@ int main(int argc, char **argv)
|
||||
AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
|
||||
AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
|
||||
//......................................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
|
||||
CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
|
||||
CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
|
||||
@@ -892,41 +893,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -994,7 +995,7 @@ int main(int argc, char **argv)
|
||||
recvMeshData_YZ = new double [recvCount_YZ];
|
||||
recvMeshData_XZ = new double [recvCount_XZ];
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1210,7 +1211,7 @@ int main(int argc, char **argv)
|
||||
CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
|
||||
CopyToDevice(Den,cDen,2*N*sizeof(double));
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
|
||||
//......................................................................
|
||||
@@ -1227,7 +1228,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
pmmc_MeshGradient(SignDist,SignDist_x,SignDist_y,SignDist_z,Nx,Ny,Nz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(SignDist_x, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(SignDist_x, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1249,7 +1250,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(SignDist_y, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(SignDist_y, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1271,7 +1272,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(SignDist_z, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(SignDist_z, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1326,42 +1327,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1429,7 +1430,7 @@ int main(int argc, char **argv)
|
||||
CopyToHost(Vel_x.get(),&Velocity[0],N*sizeof(double));
|
||||
CopyToHost(Vel_y.get(),&Velocity[N],N*sizeof(double));
|
||||
CopyToHost(Vel_z.get(),&Velocity[2*N],N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//...........................................................................
|
||||
|
||||
int timestep = 0;
|
||||
@@ -1477,7 +1478,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0) printf("Simulating {Fx,Fy,Fz} = %f, %f, %f \n", Fx,Fy,Fz);
|
||||
|
||||
//************ MAIN ITERATION LOOP ***************************************/
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
timestep = 0;
|
||||
@@ -1556,42 +1557,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1706,18 +1707,18 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
//...................................................................................
|
||||
|
||||
SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz);
|
||||
@@ -1787,42 +1788,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1866,7 +1867,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Timestep completed!
|
||||
timestep++;
|
||||
@@ -1891,7 +1892,7 @@ int main(int argc, char **argv)
|
||||
CopyToHost(Vel_x.get(),&Velocity[0],N*sizeof(double));
|
||||
CopyToHost(Vel_y.get(),&Velocity[N],N*sizeof(double));
|
||||
CopyToHost(Vel_z.get(),&Velocity[2*N],N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
if (timestep%1000 == 5){
|
||||
//...........................................................................
|
||||
@@ -1904,7 +1905,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1937,7 +1938,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Pressure
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Vel_x, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Vel_x, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1961,7 +1962,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Velocity
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Press, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Press, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1984,7 +1985,7 @@ int main(int argc, char **argv)
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Vel_y, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Vel_y, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2007,7 +2008,7 @@ int main(int argc, char **argv)
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Vel_z, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Vel_z, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2031,7 +2032,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Mean Curvature
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(MeanCurvature, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(MeanCurvature, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2056,7 +2057,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Gaussian Curvature
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(GaussCurvature, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(GaussCurvature, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2080,7 +2081,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Gradient of the phase indicator field
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_x, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_x, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2102,7 +2103,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_y, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_y, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2124,7 +2125,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_z, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_z, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2268,31 +2269,31 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
}
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
// Phase averages
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Barrier(comm);
|
||||
//.........................................................................
|
||||
// Compute the change in the total surface energy based on the defined interval
|
||||
// See McClure, Prins and Miller (2014)
|
||||
@@ -2368,7 +2369,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//************************************************************************/
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
|
||||
if (rank==0){
|
||||
@@ -2395,7 +2396,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0){
|
||||
mkdir(tmpstr,0777);
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Copy the data to the CPU
|
||||
CopyToHost(cDistEven,f_even,10*N*sizeof(double));
|
||||
@@ -2428,7 +2429,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0) printf("********************************************************\n");
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -102,8 +102,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank, nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
Utilities::setAbortBehavior( true, true, true );
|
||||
Utilities::setErrorHandlers();
|
||||
PROFILE_ENABLE(0);
|
||||
@@ -136,20 +137,20 @@ int main(int argc, char **argv)
|
||||
domain >> Ly;
|
||||
domain >> Lz;
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Check that the number of processors >= the number of ranks
|
||||
if ( rank==0 ) {
|
||||
@@ -208,7 +209,7 @@ int main(int argc, char **argv)
|
||||
// WriteLocalSolidID(LocalRankFilename, id, N);
|
||||
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
|
||||
ReadBinaryFile(LocalRankFilename, Averages.SDs.get(), N);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//.......................................................................
|
||||
//copies of data needed to perform checkpointing from cpu
|
||||
@@ -220,7 +221,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0) printf("Reading restart file! \n");
|
||||
// Read in the restart file to CPU buffers
|
||||
ReadCheckpoint(LocalRestartFile, Den, DistEven, DistOdd, N);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.........................................................................
|
||||
// Populate the arrays needed to perform averaging
|
||||
if (rank==0) printf("Populate arrays \n");
|
||||
@@ -289,9 +290,9 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
Dm.CommInit(MPI_COMM_WORLD); // Initialize communications for domains
|
||||
Dm.CommInit(comm); // Initialize communications for domains
|
||||
|
||||
MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = sum_global/Dm.Volume;
|
||||
if (rank==0) printf("Porosity = %f \n",porosity);
|
||||
|
||||
@@ -328,14 +329,14 @@ int main(int argc, char **argv)
|
||||
// BlobContainer Blobs;
|
||||
DoubleArray RecvBuffer(dimx);
|
||||
// MPI_Allreduce(&Averages.BlobAverages.get(),&Blobs.get(),1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf("Number of components is %i \n",dimy);
|
||||
|
||||
for (int b=0; b<dimy; b++){
|
||||
|
||||
MPI_Allreduce(&Averages.BlobAverages(0,b),&RecvBuffer(0),dimx,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Averages.BlobAverages(0,b),&RecvBuffer(0),dimx,MPI_DOUBLE,MPI_SUM,comm);
|
||||
for (int idx=0; idx<dimx-1; idx++) Averages.BlobAverages(idx,b)=RecvBuffer(idx);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
if (Averages.BlobAverages(0,b) > 0.0){
|
||||
double Vn,pn,awn,ans,Jwn,Kwn,lwns,cwns,trawn,trJwn;
|
||||
@@ -481,7 +482,7 @@ int main(int argc, char **argv)
|
||||
fclose(BLOBS);*/
|
||||
PROFILE_STOP("main");
|
||||
PROFILE_SAVE("BlobIdentifyParallel",false);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -266,7 +266,7 @@ int main(int argc, char **argv)
|
||||
printf("Execute blob identification algorithm... \n");
|
||||
IntArray GlobalBlobID;
|
||||
int nblobs = ComputeLocalBlobIDs( Phase, SignDist, vF, vS, GlobalBlobID );
|
||||
ReorderBlobIDs(GlobalBlobID); // This will reorder by blob size
|
||||
ReorderBlobIDs(GlobalBlobID,MPI_COMM_WORLD); // This will reorder by blob size
|
||||
printf("Identified %i blobs. Writing per-process output files. \n",nblobs);
|
||||
|
||||
int sizeLoc = nx*ny*nz;
|
||||
|
||||
@@ -49,8 +49,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank, nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
#ifdef PROFILE
|
||||
PROFILE_ENABLE(0);
|
||||
PROFILE_DISABLE_TRACE();
|
||||
@@ -99,7 +100,7 @@ int main(int argc, char **argv)
|
||||
readRankData( rank, nx+2, ny+2, nz+2, Phase, SignDist );
|
||||
|
||||
// Communication the halos
|
||||
fillHalo<double> fillData(rank_info,nx,ny,nz,1,1,1,0,1);
|
||||
fillHalo<double> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1);
|
||||
fillData.fill(Phase);
|
||||
fillData.fill(SignDist);
|
||||
|
||||
@@ -109,7 +110,7 @@ int main(int argc, char **argv)
|
||||
double vS=0.0;
|
||||
IntArray GlobalBlobID;
|
||||
int nblobs = ComputeGlobalBlobIDs(nx,ny,nz,rank_info,
|
||||
Phase,SignDist,vF,vS,GlobalBlobID);
|
||||
Phase,SignDist,vF,vS,GlobalBlobID,comm);
|
||||
if ( rank==0 ) { printf("Identified %i blobs\n",nblobs); }
|
||||
|
||||
// Write the local blob ids
|
||||
@@ -128,7 +129,7 @@ int main(int argc, char **argv)
|
||||
PROFILE_STOP("main");
|
||||
PROFILE_SAVE("BlobIdentifyParallel",false);
|
||||
#endif
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -182,8 +182,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
|
||||
printf("----------------------------------------------------------\n");
|
||||
printf("COMPUTING TCAT ANALYSIS FOR NON-WETTING PHASE FEATURES \n");
|
||||
@@ -391,7 +392,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
porosity /= (Nx*Ny*Nz*1.0);
|
||||
printf("Media porosity is %f \n",porosity);
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
/* ****************************************************************
|
||||
IDENTIFY ALL COMPONENTS FOR BOTH PHASES
|
||||
@@ -430,7 +431,7 @@ int main(int argc, char **argv)
|
||||
Averages.PrintComponents(timestep);
|
||||
|
||||
// Create the MeshDataStruct
|
||||
fillHalo<double> fillData(Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1);
|
||||
fillHalo<double> fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1);
|
||||
std::vector<IO::MeshDataStruct> meshData(1);
|
||||
meshData[0].meshName = "domain";
|
||||
meshData[0].mesh = std::shared_ptr<IO::DomainMesh>( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) );
|
||||
@@ -475,7 +476,7 @@ int main(int argc, char **argv)
|
||||
fillData.copy(Averages.Label_WP,LabelWPVar->data);
|
||||
fillData.copy(Averages.Label_NWP,LabelNWPVar->data);
|
||||
fillData.copy(Averages.PhaseID,PhaseIDVar->data);
|
||||
IO::writeData( 0, meshData, 2 );
|
||||
IO::writeData( 0, meshData, 2, comm );
|
||||
/*
|
||||
FILE *NWP_FILE;
|
||||
NWP_FILE = fopen("NWP.dat","wb");
|
||||
@@ -493,7 +494,7 @@ int main(int argc, char **argv)
|
||||
fclose(DISTANCE);
|
||||
*/
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -128,8 +128,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank, nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
|
||||
|
||||
if ( rank==0 ) {
|
||||
@@ -159,20 +160,20 @@ int main(int argc, char **argv)
|
||||
domain >> Ly;
|
||||
domain >> Lz;
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Check that the number of processors >= the number of ranks
|
||||
if ( rank==0 ) {
|
||||
@@ -204,7 +205,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
//.......................................................................
|
||||
Dm.CommInit(MPI_COMM_WORLD); // Initialize communications for domains
|
||||
Dm.CommInit(comm); // Initialize communications for domains
|
||||
//.......................................................................
|
||||
// Read in sphere pack (initialize the non-wetting phase as inside of spheres)
|
||||
if (rank==1) printf("nspheres =%i \n",nspheres);
|
||||
@@ -216,14 +217,14 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the sphere packing \n");
|
||||
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.......................................................................
|
||||
SignedDistance(Averages.Phase.get(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz,
|
||||
Dm.iproc,Dm.jproc,Dm.kproc,Dm.nprocx,Dm.nprocy,Dm.nprocz);
|
||||
@@ -289,7 +290,7 @@ int main(int argc, char **argv)
|
||||
delete [] rad;
|
||||
|
||||
} // Limit scope so variables that contain communicators will free before MPI_Finialize
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -23,10 +23,10 @@ inline double rand2()
|
||||
|
||||
|
||||
// Test if all ranks agree on a value
|
||||
bool allAgree( int x ) {
|
||||
bool allAgree( int x, MPI_Comm comm ) {
|
||||
int min, max;
|
||||
MPI_Allreduce(&x,&min,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&x,&max,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&x,&min,1,MPI_INT,MPI_MIN,comm);
|
||||
MPI_Allreduce(&x,&max,1,MPI_INT,MPI_MAX,comm);
|
||||
return min==max;
|
||||
}
|
||||
|
||||
@@ -63,10 +63,9 @@ struct bubble_struct {
|
||||
|
||||
|
||||
// Create a random set of bubles
|
||||
std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly, double Lz )
|
||||
std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly, double Lz, MPI_Comm comm )
|
||||
{
|
||||
int rank;
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
int rank = comm_rank(comm);
|
||||
std::vector<bubble_struct> bubbles(N_bubbles);
|
||||
if ( rank == 0 ) {
|
||||
double R0 = 0.2*Lx*Ly*Lz/pow((double)N_bubbles,0.333);
|
||||
@@ -81,7 +80,7 @@ std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly,
|
||||
}
|
||||
}
|
||||
size_t N_bytes = N_bubbles*sizeof(bubble_struct);
|
||||
MPI_Bcast(&bubbles[0],N_bytes,MPI_CHAR,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&bubbles[0],N_bytes,MPI_CHAR,0,comm);
|
||||
return bubbles;
|
||||
}
|
||||
|
||||
@@ -114,7 +113,7 @@ void fillBubbleData( const std::vector<bubble_struct>& bubbles, DoubleArray& Pha
|
||||
|
||||
|
||||
// Shift all of the data by the given number of cells
|
||||
void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct& rank_info )
|
||||
void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct& rank_info, MPI_Comm comm )
|
||||
{
|
||||
int nx = data.size(0)-2;
|
||||
int ny = data.size(1)-2;
|
||||
@@ -123,8 +122,8 @@ void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct
|
||||
int ngy = ny+2*abs(sy);
|
||||
int ngz = nz+2*abs(sz);
|
||||
Array<double> tmp1(nx,ny,nz), tmp2(ngx,ngy,ngz), tmp3(ngx,ngy,ngz);
|
||||
fillHalo<double> fillData1(rank_info,nx,ny,nz,1,1,1,0,1);
|
||||
fillHalo<double> fillData2(rank_info,nx,ny,nz,abs(sx),abs(sy),abs(sz),0,1);
|
||||
fillHalo<double> fillData1(comm,rank_info,nx,ny,nz,1,1,1,0,1);
|
||||
fillHalo<double> fillData2(comm,rank_info,nx,ny,nz,abs(sx),abs(sy),abs(sz),0,1);
|
||||
fillData1.copy(data,tmp1);
|
||||
fillData2.copy(tmp1,tmp2);
|
||||
fillData2.fill(tmp2);
|
||||
@@ -146,8 +145,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank, nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
PROFILE_ENABLE(1);
|
||||
PROFILE_DISABLE_TRACE();
|
||||
PROFILE_SYNCHRONIZE();
|
||||
@@ -174,11 +174,11 @@ int main(int argc, char **argv)
|
||||
// Create the dummy info
|
||||
DoubleArray Phase(nx+2,ny+2,nz+2);
|
||||
DoubleArray SignDist(nx+2,ny+2,nz+2);
|
||||
std::vector<bubble_struct> bubbles = create_bubbles(20,Lx,Ly,Lz);
|
||||
std::vector<bubble_struct> bubbles = create_bubbles(20,Lx,Ly,Lz,comm);
|
||||
fillBubbleData( bubbles, Phase, SignDist, Lx, Ly, Lz, rank_info );
|
||||
|
||||
// Communication the halos
|
||||
fillHalo<double> fillData(rank_info,nx,ny,nz,1,1,1,0,1);
|
||||
fillHalo<double> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1);
|
||||
fillData.fill(Phase);
|
||||
fillData.fill(SignDist);
|
||||
|
||||
@@ -188,7 +188,7 @@ int main(int argc, char **argv)
|
||||
double vS=0.0;
|
||||
IntArray GlobalBlobID;
|
||||
int nblobs = ComputeGlobalBlobIDs(nx,ny,nz,rank_info,
|
||||
Phase,SignDist,vF,vS,GlobalBlobID);
|
||||
Phase,SignDist,vF,vS,GlobalBlobID,comm);
|
||||
if ( rank==0 ) { printf("Identified %i blobs\n",nblobs); }
|
||||
|
||||
// Create the MeshDataStruct
|
||||
@@ -218,7 +218,7 @@ int main(int argc, char **argv)
|
||||
fillData.copy(Phase,PhaseVar->data);
|
||||
fillData.copy(SignDist,SignDistVar->data);
|
||||
fillData.copy(GlobalBlobID,BlobIDVar->data);
|
||||
IO::writeData( 0, meshData, 2 );
|
||||
IO::writeData( 0, meshData, 2, comm );
|
||||
writeIDMap(ID_map_struct(nblobs),0,"lbpm_id_map.txt");
|
||||
int save_it = 1;
|
||||
|
||||
@@ -236,18 +236,18 @@ int main(int argc, char **argv)
|
||||
int id_max = nblobs-1;
|
||||
for (int i=0; i<20; i++, save_it++) {
|
||||
// Shift all the data
|
||||
shift_data( Phase, 3, -2, 1, rank_info );
|
||||
shift_data( SignDist, 3, -2, 1, rank_info );
|
||||
shift_data( Phase, 3, -2, 1, rank_info, comm );
|
||||
shift_data( SignDist, 3, -2, 1, rank_info, comm );
|
||||
// Find blob domains
|
||||
IntArray GlobalBlobID2;
|
||||
int nblobs2 = ComputeGlobalBlobIDs(nx,ny,nz,rank_info,
|
||||
Phase,SignDist,vF,vS,GlobalBlobID2);
|
||||
Phase,SignDist,vF,vS,GlobalBlobID2,comm);
|
||||
if ( nblobs2 != nblobs ) {
|
||||
printf("Error, number of blobs changed under constant velocity (%i,%i)\n",nblobs,nblobs2);
|
||||
N_errors++;
|
||||
}
|
||||
// Identify the blob maps and renumber the ids
|
||||
ID_map_struct map = computeIDMap(GlobalBlobID,GlobalBlobID2);
|
||||
ID_map_struct map = computeIDMap(GlobalBlobID,GlobalBlobID2,comm);
|
||||
std::swap(GlobalBlobID,GlobalBlobID2);
|
||||
std::vector<BlobIDType> new_list;
|
||||
getNewIDs(map,id_max,new_list);
|
||||
@@ -270,7 +270,7 @@ int main(int argc, char **argv)
|
||||
fillData.copy(Phase,PhaseVar->data);
|
||||
fillData.copy(SignDist,SignDistVar->data);
|
||||
fillData.copy(GlobalBlobID,BlobIDVar->data);
|
||||
IO::writeData( save_it, meshData, 2 );
|
||||
IO::writeData( save_it, meshData, 2, comm );
|
||||
}
|
||||
PROFILE_STOP("constant velocity test");
|
||||
|
||||
@@ -286,15 +286,15 @@ int main(int argc, char **argv)
|
||||
velocity[i].z = bubbles[i].radius*(2*rand2()-1);
|
||||
}
|
||||
}
|
||||
MPI_Bcast(&velocity[0],bubbles.size()*sizeof(Point),MPI_CHAR,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&velocity[0],bubbles.size()*sizeof(Point),MPI_CHAR,0,comm);
|
||||
fillBubbleData( bubbles, Phase, SignDist, Lx, Ly, Lz, rank_info );
|
||||
fillData.fill(Phase);
|
||||
fillData.fill(SignDist);
|
||||
ComputeGlobalBlobIDs(nx,ny,nz,rank_info,Phase,SignDist,vF,vS,GlobalBlobID);
|
||||
ComputeGlobalBlobIDs(nx,ny,nz,rank_info,Phase,SignDist,vF,vS,GlobalBlobID,comm);
|
||||
fillData.copy(Phase,PhaseVar->data);
|
||||
fillData.copy(SignDist,SignDistVar->data);
|
||||
fillData.copy(GlobalBlobID,BlobIDVar->data);
|
||||
IO::writeData( save_it, meshData, 2 );
|
||||
IO::writeData( save_it, meshData, 2, comm );
|
||||
save_it++;
|
||||
id_max = nblobs-1;
|
||||
for (int i=0; i<25; i++, save_it++) {
|
||||
@@ -309,15 +309,15 @@ int main(int argc, char **argv)
|
||||
fillData.fill(SignDist);
|
||||
// Compute the ids
|
||||
IntArray GlobalBlobID2;
|
||||
int nblobs2 = ComputeGlobalBlobIDs(nx,ny,nz,rank_info,Phase,SignDist,vF,vS,GlobalBlobID2);
|
||||
int nblobs2 = ComputeGlobalBlobIDs(nx,ny,nz,rank_info,Phase,SignDist,vF,vS,GlobalBlobID2,comm);
|
||||
// Identify the blob maps and renumber the ids
|
||||
ID_map_struct map = computeIDMap(GlobalBlobID,GlobalBlobID2);
|
||||
ID_map_struct map = computeIDMap(GlobalBlobID,GlobalBlobID2,comm);
|
||||
std::swap(GlobalBlobID,GlobalBlobID2);
|
||||
std::vector<BlobIDType> new_list;
|
||||
getNewIDs(map,id_max,new_list);
|
||||
renumberIDs(new_list,GlobalBlobID);
|
||||
writeIDMap(map,save_it,"lbpm_id_map.txt");
|
||||
if ( !allAgree(id_max) ) {
|
||||
if ( !allAgree(id_max,comm) ) {
|
||||
if ( rank==0 )
|
||||
printf("All ranks do not agree on id_max\n");
|
||||
N_errors++;
|
||||
@@ -370,8 +370,8 @@ int main(int argc, char **argv)
|
||||
printf("\n");
|
||||
}
|
||||
}
|
||||
MPI_Bcast(&N1,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&N2,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&N1,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&N2,1,MPI_INT,0,comm);
|
||||
if ( N1!=nblobs || N2!=nblobs2 ) {
|
||||
if ( rank==0 )
|
||||
printf("Error, blob ids do not map in moving bubble test (%i,%i,%i,%i)\n",
|
||||
@@ -383,7 +383,7 @@ int main(int argc, char **argv)
|
||||
fillData.copy(Phase,PhaseVar->data);
|
||||
fillData.copy(SignDist,SignDistVar->data);
|
||||
fillData.copy(GlobalBlobID,BlobIDVar->data);
|
||||
IO::writeData( save_it, meshData, 2 );
|
||||
IO::writeData( save_it, meshData, 2, comm );
|
||||
}
|
||||
PROFILE_STOP("moving bubble test");
|
||||
|
||||
@@ -391,7 +391,7 @@ int main(int argc, char **argv)
|
||||
// Finished
|
||||
PROFILE_STOP("main");
|
||||
PROFILE_SAVE("TestBlobIdentify",false);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return N_errors;
|
||||
}
|
||||
|
||||
@@ -98,8 +98,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
Utilities::setAbortBehavior(true,true,false);
|
||||
PROFILE_ENABLE(0);
|
||||
// parallel domain size (# of sub-domains)
|
||||
@@ -204,38 +205,38 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
// **************************************************************
|
||||
double Ps = -(das-dbs)/(das+dbs);
|
||||
@@ -276,7 +277,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -449,7 +450,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -460,88 +461,88 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
*/ //**********************************************************************************
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
@@ -572,48 +573,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,comm,&req2[5]);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//......................................................................................
|
||||
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
|
||||
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
|
||||
@@ -728,7 +729,7 @@ int main(int argc, char **argv)
|
||||
AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
|
||||
AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
|
||||
//......................................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
|
||||
CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
|
||||
CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
|
||||
@@ -836,41 +837,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -938,7 +939,7 @@ int main(int argc, char **argv)
|
||||
recvMeshData_YZ = new double [recvCount_YZ];
|
||||
recvMeshData_XZ = new double [recvCount_XZ];
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1128,7 +1129,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
//...........................................................................
|
||||
@@ -1213,7 +1214,7 @@ int main(int argc, char **argv)
|
||||
CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
|
||||
CopyToDevice(Den,cDen,2*N*sizeof(double));
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1246,42 +1247,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1346,7 +1347,7 @@ int main(int argc, char **argv)
|
||||
ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
|
||||
CopyToHost(Phase.get(),Phi,N*sizeof(double));
|
||||
CopyToHost(Press.get(),Pressure,N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//...........................................................................
|
||||
|
||||
int timestep = 0;
|
||||
@@ -1429,42 +1430,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1579,18 +1580,18 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
//...................................................................................
|
||||
|
||||
SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz);
|
||||
@@ -1660,42 +1661,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1744,7 +1745,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Timestep completed!
|
||||
timestep++;
|
||||
@@ -1790,7 +1791,7 @@ int main(int argc, char **argv)
|
||||
// Fill in the halo region for the mesh gradients and curvature
|
||||
//...........................................................................
|
||||
// Pressure
|
||||
CommunicateMeshHalo(Press, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Press, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1814,7 +1815,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Mean Curvature
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(MeanCurvature, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(MeanCurvature, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1839,7 +1840,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Gaussian Curvature
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(GaussCurvature, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(GaussCurvature, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1863,7 +1864,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Gradient of the phase indicator field
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_x, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_x, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1885,7 +1886,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_y, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_y, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1907,7 +1908,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_z, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_z, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2022,27 +2023,27 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
}
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
// Phase averages
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Barrier(comm);
|
||||
//.........................................................................
|
||||
// Compute the change in the total surface energy based on the defined interval
|
||||
// See McClure, Prins and Miller (2013)
|
||||
@@ -2177,7 +2178,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
meshData[k].vars.push_back(dist);
|
||||
}
|
||||
IO::writeData( bubbleCount, meshData, 2 );
|
||||
IO::writeData( bubbleCount, meshData, 2, comm );
|
||||
#else
|
||||
fclose(WN_TRIS);
|
||||
#endif
|
||||
@@ -2188,7 +2189,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//************************************************************************/
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
if (rank==0) printf("-------------------------------------------------------------------\n");
|
||||
// Compute the walltime per timestep
|
||||
@@ -2227,7 +2228,7 @@ int main(int argc, char **argv)
|
||||
PROFILE_SAVE("TestBubble");
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
return 0;
|
||||
|
||||
@@ -158,8 +158,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -210,27 +211,27 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// **************************************************************
|
||||
// **************************************************************
|
||||
|
||||
@@ -248,7 +249,7 @@ int main(int argc, char **argv)
|
||||
printf("********************************************************\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -297,7 +298,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
//.......................................................................
|
||||
// Compute the media porosity
|
||||
@@ -315,13 +316,13 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = 1.0-sum*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
//.......................................................................
|
||||
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//...........................................................................
|
||||
|
||||
@@ -388,7 +389,7 @@ int main(int argc, char **argv)
|
||||
CopyToDevice(f_even, f_even_host, 10*dist_mem_size);
|
||||
CopyToDevice(f_odd, f_odd_host, 9*dist_mem_size);
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//*************************************************************************
|
||||
// Pack and send the D3Q19 distributions
|
||||
ScaLBL_Comm.SendD3Q19(f_even, f_odd);
|
||||
@@ -414,7 +415,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
|
||||
@@ -435,7 +436,7 @@ int main(int argc, char **argv)
|
||||
//*************************************************************************
|
||||
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Iteration completed!
|
||||
timestep++;
|
||||
//...................................................................
|
||||
@@ -464,7 +465,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9);
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
|
||||
|
||||
@@ -20,8 +20,9 @@ int main (int argc, char *argv[])
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
|
||||
int npx,npy,npz;
|
||||
int i,j,k,n;
|
||||
@@ -36,7 +37,7 @@ int main (int argc, char *argv[])
|
||||
|
||||
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
|
||||
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
TwoPhase Averages(Dm);
|
||||
int timestep=0;
|
||||
@@ -130,7 +131,7 @@ int main (int argc, char *argv[])
|
||||
return toReturn;
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
return 0;
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
|
||||
@@ -20,8 +20,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -222,7 +223,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -17,8 +17,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank, nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
|
||||
int i,j,k,n,nn;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -51,7 +52,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
nx+=2; ny+=2; nz+=2;
|
||||
N = nx*ny*nz;
|
||||
@@ -106,7 +107,7 @@ int main(int argc, char **argv)
|
||||
fwrite(Distance.get(),8,Distance.length(),DIST);
|
||||
fclose(DIST);
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
|
||||
|
||||
@@ -19,8 +19,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
{ // Limit scope so Domain can free it's communicator
|
||||
|
||||
printf("Running two-phase averaging test on %i processors \n",nprocs);
|
||||
@@ -48,7 +49,7 @@ int main(int argc, char **argv)
|
||||
|
||||
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
|
||||
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
TwoPhase Averages(Dm);
|
||||
int timestep=0;
|
||||
@@ -109,7 +110,7 @@ int main(int argc, char **argv)
|
||||
fclose(PHASE);
|
||||
}
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
} // Limit scope so Domain will free it's communicator
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
|
||||
@@ -36,8 +36,9 @@ int main(int argc, char **argv)
|
||||
{
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
Utilities::setAbortBehavior(true,true,false);
|
||||
Utilities::setErrorHandlers();
|
||||
UnitTest ut;
|
||||
@@ -131,9 +132,9 @@ int main(int argc, char **argv)
|
||||
std::vector<int> format(2,0);
|
||||
format[0] = 2;
|
||||
format[1] = 1;
|
||||
IO::writeData( 0, meshData, format[0] );
|
||||
IO::writeData( 3, meshData, format[1] );
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
IO::writeData( 0, meshData, format[0], comm );
|
||||
IO::writeData( 3, meshData, format[1], comm );
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Get a list of the timesteps
|
||||
std::vector<std::string> timesteps = IO::readTimesteps("summary.LBM");
|
||||
@@ -269,7 +270,7 @@ int main(int argc, char **argv)
|
||||
// Finished
|
||||
ut.report();
|
||||
int N_errors = ut.NumFailGlobal();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return N_errors;
|
||||
}
|
||||
|
||||
@@ -5,10 +5,9 @@
|
||||
|
||||
int main (int argc, char **argv)
|
||||
{
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
int rank = MPI_WORLD_RANK();
|
||||
int nprocs = MPI_WORLD_SIZE();
|
||||
|
||||
for (int i=0; i<nprocs; i++) {
|
||||
if ( rank==i )
|
||||
|
||||
@@ -137,6 +137,9 @@ extern "C" void CMT_MassColorCollideD3Q7(char *ID, double *A_even, double *A_odd
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
// Initialize MPI
|
||||
MPI_Init(&argc,&argv);
|
||||
|
||||
int n,N,Nx,Ny,Nz;
|
||||
|
||||
Nx = Ny = Nz = 202;
|
||||
@@ -289,18 +292,18 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
*/ //...................................................................................
|
||||
|
||||
SwapD3Q7(ID, &packed_even[0], &packed_odd[0], Nx, Ny, Nz);
|
||||
@@ -353,4 +356,9 @@ int main(int argc, char **argv)
|
||||
PHASE = fopen("Density.out","wb");
|
||||
fwrite(Den,8,2*N,PHASE);
|
||||
fclose(PHASE);
|
||||
|
||||
// Close MPI
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -102,8 +102,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -208,40 +209,40 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
RESTART_INTERVAL=interval;
|
||||
// **************************************************************
|
||||
@@ -283,7 +284,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -400,14 +401,14 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the sphere packing \n");
|
||||
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
if (rank == 0){
|
||||
// Compute the Sauter mean diameter
|
||||
@@ -421,7 +422,7 @@ int main(int argc, char **argv)
|
||||
D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx;
|
||||
printf("Sauter Mean Diameter (computed from sphere packing) = %f \n ",D);
|
||||
}
|
||||
MPI_Bcast(&D,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&D,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
//.......................................................................
|
||||
// sprintf(LocalRankString,"%05d",rank);
|
||||
@@ -481,7 +482,7 @@ int main(int argc, char **argv)
|
||||
id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0;
|
||||
//.........................................................
|
||||
sum_local = 1.0*sum;
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = porosity*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
|
||||
@@ -595,7 +596,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -606,7 +607,7 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
CommunicateSendRecvCounts( MPI_COMM_WORLD, sendtag, recvtag,
|
||||
CommunicateSendRecvCounts( comm, sendtag, recvtag,
|
||||
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy,
|
||||
rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
@@ -649,7 +650,7 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
CommunicateRecvLists( MPI_COMM_WORLD, sendtag, recvtag,
|
||||
CommunicateRecvLists( comm, sendtag, recvtag,
|
||||
sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z,
|
||||
sendList_xy, sendList_XY, sendList_xY, sendList_Xy,
|
||||
sendList_xz, sendList_XZ, sendList_xZ, sendList_Xz,
|
||||
@@ -782,7 +783,7 @@ int main(int argc, char **argv)
|
||||
AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
|
||||
AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
|
||||
//......................................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
|
||||
CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
|
||||
CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
|
||||
@@ -890,41 +891,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -992,7 +993,7 @@ int main(int argc, char **argv)
|
||||
recvMeshData_YZ = new double [recvCount_YZ];
|
||||
recvMeshData_XZ = new double [recvCount_XZ];
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1219,7 +1220,7 @@ int main(int argc, char **argv)
|
||||
CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
|
||||
CopyToDevice(Den,cDen,2*N*sizeof(double));
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
// Set up the cube list (very regular in this case due to lack of blob-ID)
|
||||
// Set up kstart, kfinish so that the reservoirs are excluded from averaging
|
||||
@@ -1284,7 +1285,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
pmmc_MeshGradient(SignDist,SignDist_x,SignDist_y,SignDist_z,Nx,Ny,Nz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(SignDist_x, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(SignDist_x, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1306,7 +1307,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(SignDist_y, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(SignDist_y, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1328,7 +1329,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(SignDist_z, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(SignDist_z, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1383,42 +1384,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1486,7 +1487,7 @@ int main(int argc, char **argv)
|
||||
CopyToHost(Vel_x.data,&Velocity[0],N*sizeof(double));
|
||||
CopyToHost(Vel_y.data,&Velocity[N],N*sizeof(double));
|
||||
CopyToHost(Vel_z.data,&Velocity[2*N],N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//...........................................................................
|
||||
|
||||
int timestep = 0;
|
||||
@@ -1499,7 +1500,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
|
||||
@@ -1596,42 +1597,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1746,18 +1747,18 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
//...................................................................................
|
||||
|
||||
SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz);
|
||||
@@ -1827,42 +1828,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1912,7 +1913,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Timestep completed!
|
||||
timestep++;
|
||||
@@ -1937,7 +1938,7 @@ int main(int argc, char **argv)
|
||||
CopyToHost(Vel_x.data,&Velocity[0],N*sizeof(double));
|
||||
CopyToHost(Vel_y.data,&Velocity[N],N*sizeof(double));
|
||||
CopyToHost(Vel_z.data,&Velocity[2*N],N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
if (timestep%1000 == 5){
|
||||
//...........................................................................
|
||||
@@ -1950,7 +1951,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -1983,7 +1984,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Pressure
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Vel_x, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Vel_x, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2007,7 +2008,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Velocity
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Press, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Press, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2030,7 +2031,7 @@ int main(int argc, char **argv)
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Vel_y, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Vel_y, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2053,7 +2054,7 @@ int main(int argc, char **argv)
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Vel_z, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Vel_z, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2077,7 +2078,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Mean Curvature
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(MeanCurvature, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(MeanCurvature, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2102,7 +2103,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Gaussian Curvature
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(GaussCurvature, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(GaussCurvature, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2126,7 +2127,7 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
// Gradient of the phase indicator field
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_x, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_x, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2148,7 +2149,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_y, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_y, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2170,7 +2171,7 @@ int main(int argc, char **argv)
|
||||
rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z,rank_xy,rank_XY,rank_xY,
|
||||
rank_Xy,rank_xz,rank_XZ,rank_xZ,rank_Xz,rank_yz,rank_YZ,rank_yZ,rank_Yz);
|
||||
//...........................................................................
|
||||
CommunicateMeshHalo(Phase_z, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Phase_z, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2442,30 +2443,30 @@ int main(int argc, char **argv)
|
||||
}
|
||||
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
// Phase averages
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Barrier(comm);
|
||||
//.........................................................................
|
||||
// Compute the change in the total surface energy based on the defined interval
|
||||
// See McClure, Prins and Miller (2014)
|
||||
@@ -2544,7 +2545,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0){
|
||||
mkdir(tmpstr,0777);
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
FILE *WN_TRIS;
|
||||
sprintf(LocalRankFilename,"%s/%s%s",tmpstr,"wn-tris.",LocalRankString);
|
||||
@@ -2689,7 +2690,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//************************************************************************/
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
if (rank==0) printf("-------------------------------------------------------------------\n");
|
||||
// Compute the walltime per timestep
|
||||
@@ -2745,7 +2746,7 @@ int main(int argc, char **argv)
|
||||
//#endif
|
||||
ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
|
||||
CopyToHost(Press.data,Pressure,N*sizeof(double));
|
||||
CommunicateMeshHalo(Press, MPI_COMM_WORLD,
|
||||
CommunicateMeshHalo(Press, comm,
|
||||
sendMeshData_x,sendMeshData_y,sendMeshData_z,sendMeshData_X,sendMeshData_Y,sendMeshData_Z,
|
||||
sendMeshData_xy,sendMeshData_XY,sendMeshData_xY,sendMeshData_Xy,sendMeshData_xz,sendMeshData_XZ,
|
||||
sendMeshData_xZ,sendMeshData_Xz,sendMeshData_yz,sendMeshData_YZ,sendMeshData_yZ,sendMeshData_Yz,
|
||||
@@ -2813,7 +2814,7 @@ int main(int argc, char **argv)
|
||||
*/ //************************************************************************/
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -204,8 +204,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -307,40 +308,40 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
RESTART_INTERVAL=interval;
|
||||
// **************************************************************
|
||||
@@ -391,7 +392,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -470,14 +471,14 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the sphere packing \n");
|
||||
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
if (rank == 0){
|
||||
// Compute the Sauter mean diameter
|
||||
@@ -491,7 +492,7 @@ int main(int argc, char **argv)
|
||||
D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx;
|
||||
printf("Sauter Mean Diameter (computed from sphere packing) = %f \n ",D);
|
||||
}
|
||||
MPI_Bcast(&D,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&D,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
//.......................................................................
|
||||
// sprintf(LocalRankString,"%05d",rank);
|
||||
@@ -530,7 +531,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
sum_local = 1.0*sum;
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = porosity*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
|
||||
@@ -552,7 +553,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
|
||||
//.........................................................
|
||||
// If pressure boundary conditions are applied remove solid
|
||||
@@ -607,7 +608,7 @@ int main(int argc, char **argv)
|
||||
*/
|
||||
// Initialize communication structures in averaging domain
|
||||
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = id[i];
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
// Set up MPI communication structures
|
||||
if (rank==0) printf ("Setting up communication control structures \n");
|
||||
@@ -714,7 +715,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -725,7 +726,7 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
CommunicateSendRecvCounts( MPI_COMM_WORLD, sendtag, recvtag,
|
||||
CommunicateSendRecvCounts( comm, sendtag, recvtag,
|
||||
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy,
|
||||
rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
@@ -768,7 +769,7 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
CommunicateRecvLists( MPI_COMM_WORLD, sendtag, recvtag,
|
||||
CommunicateRecvLists( comm, sendtag, recvtag,
|
||||
sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z,
|
||||
sendList_xy, sendList_XY, sendList_xY, sendList_Xy,
|
||||
sendList_xz, sendList_XZ, sendList_xZ, sendList_Xz,
|
||||
@@ -901,7 +902,7 @@ int main(int argc, char **argv)
|
||||
AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
|
||||
AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
|
||||
//......................................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
|
||||
CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
|
||||
CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
|
||||
@@ -1009,41 +1010,41 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -1111,7 +1112,7 @@ int main(int argc, char **argv)
|
||||
recvMeshData_YZ = new double [recvCount_YZ];
|
||||
recvMeshData_XZ = new double [recvCount_XZ];*/
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1182,7 +1183,7 @@ int main(int argc, char **argv)
|
||||
CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
|
||||
CopyToDevice(Den,cDen,2*N*sizeof(double));
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
|
||||
//.......................................................................
|
||||
@@ -1234,42 +1235,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1337,7 +1338,7 @@ int main(int argc, char **argv)
|
||||
CopyToHost(Averages.Vel_x.get(),&Velocity[0],N*sizeof(double));
|
||||
CopyToHost(Averages.Vel_y.get(),&Velocity[N],N*sizeof(double));
|
||||
CopyToHost(Averages.Vel_z.get(),&Velocity[2*N],N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//...........................................................................
|
||||
|
||||
int timestep = 0;
|
||||
@@ -1350,7 +1351,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
|
||||
@@ -1433,42 +1434,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1583,18 +1584,18 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
//...................................................................................
|
||||
|
||||
SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz);
|
||||
@@ -1664,42 +1665,42 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
@@ -1747,7 +1748,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//...................................................................................
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Timestep completed!
|
||||
timestep++;
|
||||
@@ -1773,7 +1774,7 @@ int main(int argc, char **argv)
|
||||
CopyToHost(Averages.Vel_x.get(),&Velocity[0],N*sizeof(double));
|
||||
CopyToHost(Averages.Vel_y.get(),&Velocity[N],N*sizeof(double));
|
||||
CopyToHost(Averages.Vel_z.get(),&Velocity[2*N],N*sizeof(double));
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
if (timestep%1000 == 5){
|
||||
//...........................................................................
|
||||
@@ -1832,7 +1833,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0){
|
||||
mkdir(tmpstr,0777);
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
FILE *WN_TRIS;
|
||||
sprintf(LocalRankFilename,"%s/%s%s",tmpstr,"wn-tris.",LocalRankString);
|
||||
@@ -2008,7 +2009,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//************************************************************************/
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
if (rank==0) printf("-------------------------------------------------------------------\n");
|
||||
// Compute the walltime per timestep
|
||||
@@ -2104,7 +2105,7 @@ int main(int argc, char **argv)
|
||||
*/ //************************************************************************/
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -47,8 +47,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -88,19 +89,19 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
}
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
|
||||
TwoPhase Averages(Dm);
|
||||
@@ -121,16 +122,16 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
|
||||
ReadBinaryFile(LocalRankFilename, Averages.SDs.get(), N);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// sprintf(LocalRankFilename,"%s%s","Pressure.",LocalRankString);
|
||||
//ReadBinaryFile(LocalRankFilename, Averages.Press.get(), N);
|
||||
//MPI_Barrier(MPI_COMM_WORLD);
|
||||
//MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//.......................................................................
|
||||
sprintf(LocalRankFilename,"%s%s","Label_NWP.",LocalRankString);
|
||||
ReadBlobFile(LocalRankFilename, Averages.Label_NWP.get(), N);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Label_NWP set." << endl;
|
||||
|
||||
//.......................................................................
|
||||
@@ -143,7 +144,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0) printf("Reading restart file! \n");
|
||||
// Read in the restart file to CPU buffers
|
||||
ReadCheckpoint(LocalRestartFile, Den, DistEven, DistOdd, N);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.........................................................................
|
||||
// Populate the arrays needed to perform averaging
|
||||
if (rank==0) printf("Populate arrays \n");
|
||||
@@ -197,10 +198,10 @@ int main(int argc, char **argv)
|
||||
delete [] DistEven;
|
||||
delete [] DistOdd;
|
||||
|
||||
MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = sum_global/Dm.Volume;
|
||||
if (rank==0) printf("Porosity = %f \n",porosity);
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
for (int i=0; i<N; i++) Averages.SDs(i) -= 1.0; // map the distance
|
||||
|
||||
double beta = 0.95;
|
||||
@@ -227,14 +228,14 @@ int main(int argc, char **argv)
|
||||
// BlobContainer Blobs;
|
||||
DoubleArray RecvBuffer(dimx);
|
||||
// MPI_Allreduce(&Averages.ComponentAverages_NWP.get(),&Blobs.get(),1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank==0) printf("All ranks passed gate \n");
|
||||
|
||||
for (int b=0; b<(int)Averages.ComponentAverages_NWP.size(1); b++){
|
||||
|
||||
MPI_Allreduce(&Averages.ComponentAverages_NWP(0,b),&RecvBuffer(0),dimx,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&Averages.ComponentAverages_NWP(0,b),&RecvBuffer(0),dimx,MPI_DOUBLE,MPI_SUM,comm);
|
||||
for (int idx=0; idx<dimx-1; idx++) Averages.ComponentAverages_NWP(idx,b)=RecvBuffer(idx);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (Averages.ComponentAverages_NWP(0,b) > 0.0){
|
||||
double Vn,pn,awn,ans,Jwn,Kwn,lwns,cwns,trawn,trJwn;
|
||||
Vn = Averages.ComponentAverages_NWP(1,b);
|
||||
@@ -298,10 +299,10 @@ int main(int argc, char **argv)
|
||||
double Length=1.0;
|
||||
if (rank==0) WriteBlobStates(Averages,Length,porosity);
|
||||
|
||||
//MPI_Barrier(MPI_COMM_WORLD);
|
||||
//MPI_Barrier(comm);
|
||||
//printf("Exit, rank=%i \n",rank);
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -19,8 +19,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -75,20 +76,20 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// **************************************************************
|
||||
if (nprocs != nprocx*nprocy*nprocz){
|
||||
@@ -107,7 +108,7 @@ int main(int argc, char **argv)
|
||||
|
||||
// Initialized domain and averaging framework for Two-Phase Flow
|
||||
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
TwoPhase Averages(Dm);
|
||||
|
||||
InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc,
|
||||
@@ -115,7 +116,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -189,7 +190,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
|
||||
//.........................................................
|
||||
// don't perform computations at the eight corners
|
||||
@@ -208,7 +209,7 @@ int main(int argc, char **argv)
|
||||
fclose(ID);
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -101,10 +101,15 @@ inline void ZeroHalo(double *Data, int Nx, int Ny, int Nz)
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
int provided_thread_support=-1;
|
||||
//MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support);
|
||||
MPI_Init_thread(&argc,&argv,MPI_THREAD_SINGLE,&provided_thread_support);
|
||||
if ( provided_thread_support<MPI_THREAD_MULTIPLE )
|
||||
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
|
||||
MPI_Comm comm;
|
||||
MPI_Comm_dup(MPI_COMM_WORLD,&comm);
|
||||
int rank = comm_rank(comm);
|
||||
int nprocs = comm_size(comm);
|
||||
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
|
||||
|
||||
// parallel domain size (# of sub-domains)
|
||||
@@ -214,42 +219,42 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&InitialCondition,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
|
||||
// Get the rank info
|
||||
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
RESTART_INTERVAL=interval;
|
||||
// **************************************************************
|
||||
@@ -310,7 +315,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -351,7 +356,7 @@ int main(int argc, char **argv)
|
||||
// WriteLocalSolidID(LocalRankFilename, id, N);
|
||||
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
|
||||
ReadBinaryFile(LocalRankFilename, Averages->SDs.get(), N);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
|
||||
//.......................................................................
|
||||
@@ -423,8 +428,8 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
// MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
// MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = pore_vol*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
//.........................................................
|
||||
@@ -459,7 +464,7 @@ int main(int argc, char **argv)
|
||||
|
||||
// Initialize communication structures in averaging domain
|
||||
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = id[i];
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
//...........................................................................
|
||||
if (rank==0) printf ("Create ScaLBL_Communicator \n");
|
||||
@@ -561,14 +566,14 @@ int main(int argc, char **argv)
|
||||
delete [] cDen;
|
||||
delete [] cDistEven;
|
||||
delete [] cDistOdd;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
}
|
||||
|
||||
//......................................................................
|
||||
InitD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz);
|
||||
InitD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz);
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.......................................................................
|
||||
// Once phase has been initialized, map solid to account for 'smeared' interface
|
||||
//for (i=0; i<N; i++) Averages->SDs(i) -= (1.0); //
|
||||
@@ -587,7 +592,7 @@ int main(int argc, char **argv)
|
||||
ScaLBL_Comm.SendHalo(Phi);
|
||||
ScaLBL_Comm.RecvHalo(Phi);
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//*************************************************************************
|
||||
|
||||
if (rank==0 && BoundaryCondition==1){
|
||||
@@ -660,26 +665,33 @@ int main(int argc, char **argv)
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
|
||||
err = 1.0;
|
||||
double sat_w_previous = 1.01; // slightly impossible value!
|
||||
if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol);
|
||||
// Create the thread pool
|
||||
int N_threads = 0;
|
||||
if ( provided_thread_support<MPI_THREAD_MULTIPLE )
|
||||
N_threads = 0;
|
||||
if ( N_threads > 0 ) {
|
||||
// Set the affinity
|
||||
int N_procs = ThreadPool::getNumberOfProcessors();
|
||||
std::vector<int> procs(N_procs);
|
||||
for (int i=0; i<N_procs; i++)
|
||||
procs[i] = i;
|
||||
ThreadPool::setProcessAffinity(procs);
|
||||
}
|
||||
ThreadPool tpool(N_threads);
|
||||
//************ MAIN ITERATION LOOP ***************************************/
|
||||
PROFILE_START("Loop");
|
||||
int N_procs = ThreadPool::getNumberOfProcessors();
|
||||
std::vector<int> procs(N_procs);
|
||||
for (int i=0; i<N_procs; i++)
|
||||
procs[i] = i;
|
||||
ThreadPool::setProcessAffinity(procs);
|
||||
int timestep = -1;
|
||||
AnalysisWaitIdStruct work_ids;
|
||||
ThreadPool tpool(0);
|
||||
BlobIDstruct last_ids, last_index;
|
||||
BlobIDList last_id_map;
|
||||
writeIDMap(ID_map_struct(),0,id_map_filename);
|
||||
AnalysisWaitIdStruct work_ids;
|
||||
while (timestep < timestepMax && err > tol ) {
|
||||
PROFILE_START("Update");
|
||||
|
||||
@@ -704,7 +716,7 @@ int main(int argc, char **argv)
|
||||
//*************************************************************************
|
||||
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//*************************************************************************
|
||||
// Swap the distributions for momentum transport
|
||||
//*************************************************************************
|
||||
@@ -712,7 +724,7 @@ int main(int argc, char **argv)
|
||||
//*************************************************************************
|
||||
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//*************************************************************************
|
||||
// Wait for communications to complete and unpack the distributions
|
||||
ScaLBL_Comm.RecvD3Q19(f_even, f_odd);
|
||||
@@ -729,7 +741,7 @@ int main(int argc, char **argv)
|
||||
SwapD3Q7(ID, B_even, B_odd, Nx, Ny, Nz);
|
||||
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
//*************************************************************************
|
||||
// Wait for communication and unpack the D3Q7 distributions
|
||||
@@ -744,7 +756,7 @@ int main(int argc, char **argv)
|
||||
// Compute the phase indicator field
|
||||
//*************************************************************************
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
ComputePhi(ID, Phi, Den, N);
|
||||
//*************************************************************************
|
||||
@@ -778,7 +790,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//...................................................................................
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
PROFILE_STOP("Update");
|
||||
|
||||
// Timestep completed!
|
||||
@@ -794,7 +806,7 @@ int main(int argc, char **argv)
|
||||
PROFILE_STOP("Loop");
|
||||
//************************************************************************
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
if (rank==0) printf("-------------------------------------------------------------------\n");
|
||||
// Compute the walltime per timestep
|
||||
@@ -823,7 +835,7 @@ int main(int argc, char **argv)
|
||||
CopyToHost(Averages->Phase.get(),Phi,N*sizeof(double));
|
||||
*/
|
||||
// Create the MeshDataStruct
|
||||
fillHalo<double> fillData(Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1);
|
||||
fillHalo<double> fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1);
|
||||
std::vector<IO::MeshDataStruct> meshData(1);
|
||||
meshData[0].meshName = "domain";
|
||||
meshData[0].mesh = std::shared_ptr<IO::DomainMesh>( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) );
|
||||
@@ -855,7 +867,7 @@ int main(int argc, char **argv)
|
||||
fillData.copy(Averages->SDn,PhaseVar->data);
|
||||
fillData.copy(Averages->SDs,SignDistVar->data);
|
||||
fillData.copy(Averages->Label_NWP,BlobIDVar->data);
|
||||
IO::writeData( 0, meshData, 2 );
|
||||
IO::writeData( 0, meshData, 2, comm );
|
||||
|
||||
/* Averages->WriteSurfaces(0);
|
||||
|
||||
@@ -884,8 +896,9 @@ int main(int argc, char **argv)
|
||||
PROFILE_STOP("Main");
|
||||
PROFILE_SAVE("lbpm_color_simulator",1);
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
} // Limit scope so variables that contain communicators will free before MPI_Finialize
|
||||
MPI_Comm_free(&comm);
|
||||
MPI_Finalize();
|
||||
}
|
||||
|
||||
|
||||
@@ -53,17 +53,19 @@ public:
|
||||
ThreadPool::WorkItem::d_state = 1; // Change state to in progress
|
||||
// Compute the global blob id and compare to the previous version
|
||||
PROFILE_START("Identify blobs and maps",1);
|
||||
MPI_Comm newcomm;
|
||||
MPI_Comm_dup(MPI_COMM_WORLD,&newcomm);
|
||||
double vF = 0.0;
|
||||
double vS = 0.0;
|
||||
IntArray& ids = new_index->second;
|
||||
new_index->first = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,*phase,dist,vF,vS,ids);
|
||||
new_index->first = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,*phase,dist,vF,vS,ids,newcomm);
|
||||
static int max_id = -1;
|
||||
new_id->first = new_index->first;
|
||||
new_id->second = new_index->second;
|
||||
if ( last_id!=NULL ) {
|
||||
// Compute the timestep-timestep map
|
||||
const IntArray& old_ids = last_id->second;
|
||||
ID_map_struct map = computeIDMap(old_ids,ids);
|
||||
ID_map_struct map = computeIDMap(old_ids,ids,newcomm);
|
||||
// Renumber the current timestep's ids
|
||||
getNewIDs(map,max_id,*new_list);
|
||||
renumberIDs(*new_list,new_id->second);
|
||||
@@ -74,6 +76,7 @@ public:
|
||||
getNewIDs(map,max_id,*new_list);
|
||||
writeIDMap(map,timestep,id_map_filename);
|
||||
}
|
||||
MPI_Comm_free(&newcomm);
|
||||
PROFILE_STOP("Identify blobs and maps",1);
|
||||
ThreadPool::WorkItem::d_state = 2; // Change state to finished
|
||||
}
|
||||
@@ -240,8 +243,7 @@ void run_analysis( int timestep, int restart_interval,
|
||||
|
||||
// Spawn a thread to write the restart file
|
||||
if ( (type&CreateRestart) != 0 ) {
|
||||
int rank;
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
int rank = MPI_WORLD_RANK();
|
||||
if (pBC) {
|
||||
//err = fabs(sat_w - sat_w_previous);
|
||||
//sat_w_previous = sat_w;
|
||||
|
||||
@@ -136,8 +136,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -189,21 +190,21 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&ndiscs,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&ndiscs,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// **************************************************************
|
||||
if (argc > 1) depth=atoi(argv[1]);
|
||||
@@ -221,7 +222,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -274,13 +275,13 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the disc packing \n");
|
||||
if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0){
|
||||
cout << "Domain set." << endl;
|
||||
printf("************ \n");
|
||||
@@ -347,7 +348,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
sum_local = 1.0*sum;
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = porosity*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
|
||||
@@ -363,7 +364,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
|
||||
//.........................................................
|
||||
// don't perform computations at the eight corners
|
||||
@@ -378,7 +379,7 @@ int main(int argc, char **argv)
|
||||
//......................................................................
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -101,8 +101,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -185,32 +186,32 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
RESTART_INTERVAL=interval;
|
||||
// **************************************************************
|
||||
@@ -247,7 +248,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -288,7 +289,7 @@ int main(int argc, char **argv)
|
||||
// WriteLocalSolidID(LocalRankFilename, id, N);
|
||||
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
|
||||
ReadBinaryFile(LocalRankFilename, Averages.SDs.get(), N);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
|
||||
//.......................................................................
|
||||
@@ -338,8 +339,8 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
// MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
// MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = pore_vol*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
//.........................................................
|
||||
@@ -373,7 +374,7 @@ int main(int argc, char **argv)
|
||||
//.........................................................
|
||||
// Initialize communication structures in averaging domain
|
||||
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = id[i];
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
//...........................................................................
|
||||
if (rank==0) printf ("Create ScaLBL_Communicator \n");
|
||||
@@ -465,7 +466,7 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
//.........................................
|
||||
|
||||
@@ -503,7 +504,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//...................................................................................
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Timestep completed!
|
||||
timestep++;
|
||||
@@ -562,7 +563,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//************************************************************************/
|
||||
DeviceBarrier();
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
if (rank==0) printf("-------------------------------------------------------------------\n");
|
||||
// Compute the walltime per timestep
|
||||
@@ -578,7 +579,7 @@ int main(int argc, char **argv)
|
||||
if (rank==0) printf("********************************************************\n");
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -54,8 +54,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank, nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
|
||||
int InitialWetting;
|
||||
double Saturation;
|
||||
@@ -96,20 +97,20 @@ int main(int argc, char **argv)
|
||||
domain >> Lz;
|
||||
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Check that the number of processors >= the number of ranks
|
||||
if ( rank==0 ) {
|
||||
@@ -140,7 +141,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
DoubleArray SignDist(nx,ny,nz);
|
||||
// Read the signed distance from file
|
||||
@@ -164,9 +165,9 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
// total Global is the number of nodes in the pore-space
|
||||
MPI_Allreduce(&count,&totalGlobal,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&count,&totalGlobal,1,MPI_INT,MPI_SUM,comm);
|
||||
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
int iproc = Dm.iproc;
|
||||
int jproc = Dm.jproc;
|
||||
int kproc = Dm.kproc;
|
||||
@@ -212,12 +213,12 @@ int main(int argc, char **argv)
|
||||
sizeY = SizeY[bin];
|
||||
sizeZ = SizeZ[bin];
|
||||
}
|
||||
MPI_Bcast(&x,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&y,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&z,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&sizeX,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&sizeY,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&sizeZ,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&x,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&y,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&z,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&sizeX,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&sizeY,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&sizeZ,1,MPI_INT,0,comm);
|
||||
|
||||
//if (rank==0) printf("Broadcast block at %i,%i,%i \n",x,y,z);
|
||||
|
||||
@@ -265,7 +266,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm);
|
||||
sat = float(countGlobal)/totalGlobal;
|
||||
//if (rank==0) printf("New count=%i\n",countGlobal);
|
||||
//if (rank==0) printf("New saturation=%f\n",sat);
|
||||
@@ -342,41 +343,41 @@ int main(int argc, char **argv)
|
||||
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x,sendtag,
|
||||
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X,sendtag,
|
||||
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y,sendtag,
|
||||
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y,sendtag,
|
||||
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z,sendtag,
|
||||
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z,sendtag,
|
||||
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy,sendtag,
|
||||
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY,sendtag,
|
||||
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy,sendtag,
|
||||
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY,sendtag,
|
||||
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz,sendtag,
|
||||
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ,sendtag,
|
||||
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz,sendtag,
|
||||
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ,sendtag,
|
||||
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz,sendtag,
|
||||
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ,sendtag,
|
||||
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz,sendtag,
|
||||
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ,sendtag,
|
||||
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
//......................................................................................
|
||||
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
|
||||
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);
|
||||
@@ -409,7 +410,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm);
|
||||
sat = float(countGlobal)/totalGlobal;
|
||||
if (rank==0) printf("Final saturation=%f\n",sat);
|
||||
|
||||
@@ -418,7 +419,7 @@ int main(int argc, char **argv)
|
||||
fwrite(id,1,N,ID);
|
||||
fclose(ID);
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -18,8 +18,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank, nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
|
||||
//.......................................................................
|
||||
// Reading the domain information file
|
||||
@@ -60,28 +61,28 @@ int main(int argc, char **argv)
|
||||
image >> zStart;
|
||||
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Computational domain
|
||||
//.................................................
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&xStart,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&yStart,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&zStart,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&xStart,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&yStart,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&zStart,1,MPI_INT,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Check that the number of processors >= the number of ranks
|
||||
if ( rank==0 ) {
|
||||
@@ -103,7 +104,7 @@ int main(int argc, char **argv)
|
||||
fclose(SEGDAT);
|
||||
printf("Read segmented data from %s \n",Filename);
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Get the rank info
|
||||
int N = (nx+2)*(ny+2)*(nz+2);
|
||||
@@ -116,7 +117,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
// number of sites to use for periodic boundary condition transition zone
|
||||
int z_transition_size = (nprocz*nz - (Nz - zStart))/2;
|
||||
@@ -163,7 +164,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
else{
|
||||
printf("Sending data to process %i \n", rnk);
|
||||
MPI_Send(tmp,N,MPI_CHAR,rnk,15,MPI_COMM_WORLD);
|
||||
MPI_Send(tmp,N,MPI_CHAR,rnk,15,comm);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -172,9 +173,9 @@ int main(int argc, char **argv)
|
||||
else{
|
||||
// Recieve the subdomain from rank = 0
|
||||
printf("Ready to recieve data %i at process %i \n", N,rank);
|
||||
MPI_Recv(Dm.id,N,MPI_CHAR,0,15,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
|
||||
MPI_Recv(Dm.id,N,MPI_CHAR,0,15,comm,MPI_STATUS_IGNORE);
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
nx+=2; ny+=2; nz+=2;
|
||||
int count = 0;
|
||||
@@ -226,7 +227,7 @@ int main(int argc, char **argv)
|
||||
fwrite(symid,1,N,SYMID);
|
||||
fclose(SYMID);
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
|
||||
|
||||
@@ -31,8 +31,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank, nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
|
||||
//.......................................................................
|
||||
// Reading the domain information file
|
||||
@@ -73,20 +74,20 @@ int main(int argc, char **argv)
|
||||
image >> zStart;
|
||||
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Computational domain
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// Check that the number of processors >= the number of ranks
|
||||
if ( rank==0 ) {
|
||||
@@ -108,7 +109,7 @@ int main(int argc, char **argv)
|
||||
fread(Dm.id,1,N,ID);
|
||||
fclose(ID);
|
||||
// Initialize the domain and communication
|
||||
Dm.CommInit(MPI_COMM_WORLD);
|
||||
Dm.CommInit(comm);
|
||||
|
||||
nx+=2; ny+=2; nz+=2;
|
||||
int count = 0;
|
||||
@@ -209,7 +210,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
|
||||
// Create the MeshDataStruct
|
||||
fillHalo<double> fillData(Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1);
|
||||
fillHalo<double> fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1);
|
||||
std::vector<IO::MeshDataStruct> meshData(1);
|
||||
meshData[0].meshName = "domain";
|
||||
meshData[0].mesh = std::shared_ptr<IO::DomainMesh>( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) );
|
||||
@@ -235,7 +236,7 @@ int main(int argc, char **argv)
|
||||
fillData.copy(Averages.SDn,PhaseVar->data);
|
||||
fillData.copy(Averages.SDs,SignDistVar->data);
|
||||
fillData.copy(Averages.Label_NWP,BlobIDVar->data);
|
||||
IO::writeData( 0, meshData, 2 );
|
||||
IO::writeData( 0, meshData, 2, comm );
|
||||
|
||||
// sprintf(LocalRankFilename,"Phase.%05i",rank);
|
||||
// FILE *PHASE = fopen(LocalRankFilename,"wb");
|
||||
@@ -261,7 +262,7 @@ int main(int argc, char **argv)
|
||||
Averages.SortBlobs();
|
||||
Averages.PrintComponents(timestep);
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
|
||||
|
||||
@@ -28,8 +28,9 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -80,21 +81,21 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
//.................................................
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
//.................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
// **************************************************************
|
||||
|
||||
@@ -110,7 +111,7 @@ int main(int argc, char **argv)
|
||||
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
|
||||
rank_yz, rank_YZ, rank_yZ, rank_Yz );
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
|
||||
Nz += 2;
|
||||
Nx = Ny = Nz; // Cubic domain
|
||||
@@ -160,14 +161,14 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the sphere packing \n");
|
||||
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
|
||||
//...........................................................................
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
if (rank == 0){
|
||||
// Compute the Sauter mean diameter
|
||||
@@ -181,7 +182,7 @@ int main(int argc, char **argv)
|
||||
D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx;
|
||||
printf("Sauter Mean Diameter (computed from sphere packing) = %f \n",D);
|
||||
}
|
||||
MPI_Bcast(&D,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
|
||||
MPI_Bcast(&D,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
//.......................................................................
|
||||
SignedDistance(SignDist.get(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz,
|
||||
@@ -214,7 +215,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
sum_local = 1.0*sum;
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = porosity*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
|
||||
@@ -230,7 +231,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
|
||||
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
|
||||
//.........................................................
|
||||
// don't perform computations at the eight corners
|
||||
@@ -245,7 +246,7 @@ int main(int argc, char **argv)
|
||||
//......................................................................
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -221,7 +221,7 @@ int testHalo( MPI_Comm comm, int nprocx, int nprocy, int nprocz, int depth )
|
||||
}
|
||||
|
||||
// Communicate the halo
|
||||
fillHalo<TYPE> fillData(rank_info,nx,ny,nz,1,1,1,0,depth);
|
||||
fillHalo<TYPE> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,depth);
|
||||
fillData.fill(array);
|
||||
|
||||
// Check the results
|
||||
@@ -257,41 +257,42 @@ int main(int argc, char **argv)
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
|
||||
// Run the test with different domains
|
||||
int N_errors = 0;
|
||||
N_errors += test_communication( MPI_COMM_WORLD, nprocs, 1, 1 );
|
||||
N_errors += test_communication( MPI_COMM_WORLD, 1, nprocs, 1 );
|
||||
N_errors += test_communication( MPI_COMM_WORLD, 1, 1, nprocs );
|
||||
N_errors += test_communication( comm, nprocs, 1, 1 );
|
||||
N_errors += test_communication( comm, 1, nprocs, 1 );
|
||||
N_errors += test_communication( comm, 1, 1, nprocs );
|
||||
if ( nprocs==4 ) {
|
||||
N_errors += test_communication( MPI_COMM_WORLD, 2, 2, 1 );
|
||||
N_errors += test_communication( MPI_COMM_WORLD, 2, 1, 2 );
|
||||
N_errors += test_communication( MPI_COMM_WORLD, 1, 2, 2 );
|
||||
N_errors += test_communication( comm, 2, 2, 1 );
|
||||
N_errors += test_communication( comm, 2, 1, 2 );
|
||||
N_errors += test_communication( comm, 1, 2, 2 );
|
||||
}
|
||||
|
||||
// Run the halo tests with different domains
|
||||
N_errors += testHalo<int>( MPI_COMM_WORLD, nprocs, 1, 1, 1 );
|
||||
N_errors += testHalo<int>( MPI_COMM_WORLD, 1, nprocs, 1, 1 );
|
||||
N_errors += testHalo<int>( MPI_COMM_WORLD, 1, 1, nprocs, 1 );
|
||||
N_errors += testHalo<double>( MPI_COMM_WORLD, nprocs, 1, 1, 3 );
|
||||
N_errors += testHalo<double>( MPI_COMM_WORLD, 1, nprocs, 1, 3 );
|
||||
N_errors += testHalo<double>( MPI_COMM_WORLD, 1, 1, nprocs, 3 );
|
||||
N_errors += testHalo<int>( comm, nprocs, 1, 1, 1 );
|
||||
N_errors += testHalo<int>( comm, 1, nprocs, 1, 1 );
|
||||
N_errors += testHalo<int>( comm, 1, 1, nprocs, 1 );
|
||||
N_errors += testHalo<double>( comm, nprocs, 1, 1, 3 );
|
||||
N_errors += testHalo<double>( comm, 1, nprocs, 1, 3 );
|
||||
N_errors += testHalo<double>( comm, 1, 1, nprocs, 3 );
|
||||
if ( nprocs==4 ) {
|
||||
N_errors += testHalo<int>( MPI_COMM_WORLD, 2, 2, 1, 1 );
|
||||
N_errors += testHalo<int>( MPI_COMM_WORLD, 2, 1, 2, 1 );
|
||||
N_errors += testHalo<int>( MPI_COMM_WORLD, 1, 2, 2, 1 );
|
||||
N_errors += testHalo<int>( comm, 2, 2, 1, 1 );
|
||||
N_errors += testHalo<int>( comm, 2, 1, 2, 1 );
|
||||
N_errors += testHalo<int>( comm, 1, 2, 2, 1 );
|
||||
}
|
||||
if ( nprocs==8 ) {
|
||||
N_errors += testHalo<int>( MPI_COMM_WORLD, 2, 2, 2, 1 );
|
||||
N_errors += testHalo<int>( comm, 2, 2, 2, 1 );
|
||||
}
|
||||
|
||||
// Finished
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Barrier(comm);
|
||||
int N_errors_global=0;
|
||||
MPI_Allreduce( &N_errors, &N_errors_global, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Allreduce( &N_errors, &N_errors_global, 1, MPI_INT, MPI_SUM, comm );
|
||||
MPI_Barrier(comm);
|
||||
MPI_Finalize();
|
||||
if ( rank==0 ) {
|
||||
if ( N_errors_global==0 )
|
||||
|
||||
@@ -45,7 +45,8 @@ int main(int argc, char *argv[])
|
||||
{
|
||||
int rank = 0;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
UnitTest ut;
|
||||
Utilities::setAbortBehavior( true, true, true );
|
||||
|
||||
|
||||
Reference in New Issue
Block a user