Over-wrote common/ with version from ScaLBL (AA model refactor)
This commit is contained in:
James E McClure
parent
282c296935
commit
db8a1bfba9
@ -4,7 +4,7 @@
|
||||
#include "common/Communication.h"
|
||||
#include "common/MPI_Helpers.h"
|
||||
#include "common/Utilities.h"
|
||||
#include "ProfilerApp.h"
|
||||
//#include "ProfilerApp.h"
|
||||
|
||||
|
||||
/********************************************************
|
||||
@ -104,7 +104,7 @@ fillHalo<TYPE>::~fillHalo( )
|
||||
template<class TYPE>
|
||||
void fillHalo<TYPE>::fill( Array<TYPE>& data )
|
||||
{
|
||||
PROFILE_START("fillHalo::fill",1);
|
||||
//PROFILE_START("fillHalo::fill",1);
|
||||
int depth2 = data.size(3);
|
||||
ASSERT((int)data.size(0)==nx+2*ngx);
|
||||
ASSERT((int)data.size(1)==ny+2*ngy);
|
||||
@ -156,7 +156,7 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
|
||||
}
|
||||
}
|
||||
}
|
||||
PROFILE_STOP("fillHalo::fill",1);
|
||||
//PROFILE_STOP("fillHalo::fill",1);
|
||||
}
|
||||
template<class TYPE>
|
||||
void fillHalo<TYPE>::pack( const Array<TYPE>& data, int i0, int j0, int k0, TYPE *buffer )
|
||||
@ -207,7 +207,7 @@ template<class TYPE>
|
||||
template<class TYPE1, class TYPE2>
|
||||
void fillHalo<TYPE>::copy( const Array<TYPE1>& src, Array<TYPE2>& dst )
|
||||
{
|
||||
PROFILE_START("fillHalo::copy",1);
|
||||
//PROFILE_START("fillHalo::copy",1);
|
||||
ASSERT( (int)src.size(0)==nx || (int)src.size(0)==nx+2*ngx );
|
||||
ASSERT( (int)dst.size(0)==nx || (int)dst.size(0)==nx+2*ngx );
|
||||
bool src_halo = (int)src.size(0)==nx+2*ngx;
|
||||
@ -254,7 +254,7 @@ void fillHalo<TYPE>::copy( const Array<TYPE1>& src, Array<TYPE2>& dst )
|
||||
}
|
||||
fill(dst);
|
||||
}
|
||||
PROFILE_STOP("fillHalo::copy",1);
|
||||
//PROFILE_STOP("fillHalo::copy",1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -15,13 +15,8 @@
|
||||
#include "common/MPI_Helpers.h"
|
||||
#include "common/Communication.h"
|
||||
|
||||
static int MAX_BLOB_COUNT=50;
|
||||
|
||||
using namespace std;
|
||||
|
||||
|
||||
|
||||
|
||||
// Reading the domain information file
|
||||
void read_domain( int rank, int nprocs, MPI_Comm comm,
|
||||
int& nprocx, int& nprocy, int& nprocz, int& nx, int& ny, int& nz,
|
||||
@ -59,8 +54,8 @@ void read_domain( int rank, int nprocs, MPI_Comm comm,
|
||||
|
||||
|
||||
/********************************************************
|
||||
* Constructor/Destructor *
|
||||
********************************************************/
|
||||
* Constructor/Destructor *
|
||||
********************************************************/
|
||||
Domain::Domain(int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
|
||||
double lx, double ly, double lz, int BC):
|
||||
Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), nprocx(0), nprocy(0), nprocz(0),
|
||||
@ -104,8 +99,6 @@ Domain::Domain(int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
|
||||
N = Nx*Ny*Nz;
|
||||
id = new char[N];
|
||||
memset(id,0,N);
|
||||
BlobLabel.resize(Nx,Ny,Nz);
|
||||
BlobGraph.resize(18,MAX_BLOB_COUNT,MAX_BLOB_COUNT);
|
||||
BoundaryCondition = BC;
|
||||
rank_info=RankInfoStruct(rank,nprocx,nprocy,nprocz);
|
||||
}
|
||||
@ -163,63 +156,9 @@ Domain::~Domain()
|
||||
}
|
||||
|
||||
|
||||
|
||||
int Domain::VoxelConnection(int n)
|
||||
{
|
||||
const int d[26][3] = {{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
|
||||
{1,1,0},{1,-1,0},{-1,1,0},{-1,-1,0},{1,0,1},{-1,0,1},
|
||||
{1,0,-1},{-1,0,-1},{0,1,1},{0,-1,1},{0,1,-1},{0,-1,-1},
|
||||
{1,1,1},{1,1,-1},{1,-1,1},{1,-1,-1},{-1,1,1},{-1,1,-1},
|
||||
{-1,-1,1},{-1,-1,-1}}; // directions to neighbors
|
||||
|
||||
int returnVal = -1;
|
||||
int x,y,z;
|
||||
// Get the 3-D indices
|
||||
x = n%Nx;
|
||||
y = (n/Nx)%Ny;
|
||||
z = n/(Nx*Ny);
|
||||
int nodx,nody,nodz;
|
||||
for (int p=0;p<26;p++){
|
||||
nodx=x+d[p][0];
|
||||
// Get the neighbor and guarantee it is in the domain
|
||||
if (nodx < 0 ){ nodx = 0; }
|
||||
if (nodx > Nx-1 ){ nodx = Nx-1; }
|
||||
nody=y+d[p][1];
|
||||
if (nody < 0 ){ nody = 0; }
|
||||
if (nody > Ny-1 ){ nody = Ny-1; }
|
||||
nodz=z+d[p][2];
|
||||
if (nodz < 0 ){ nodz = 0; }
|
||||
if (nodz > Nz-1 ){ nodz = Nz-1; }
|
||||
|
||||
if (BlobLabel(nodx,nody,nodz) > returnVal ) returnVal = BlobLabel(nodx,nody,nodz);
|
||||
}
|
||||
return returnVal;
|
||||
}
|
||||
|
||||
void Domain::getBlobConnections(int * List, int count, int neighbor, int direction){
|
||||
|
||||
int idx,n,localValue,neighborValue;
|
||||
int x,y,z;
|
||||
for (idx=0; idx<count; idx++){
|
||||
n = List[idx];
|
||||
// Get the 3-D indices
|
||||
x = n%Nx;
|
||||
y = (n/Nx)%Ny;
|
||||
z = n/(Nx*Ny);
|
||||
neighborValue = BlobLabel(x,y,z);
|
||||
if (neighborValue > -1){
|
||||
localValue = VoxelConnection(n);
|
||||
printf("Blob (%i,%i) connects to neighbor blob (%i,%i)", localValue, rank, neighborValue, neighbor);
|
||||
BlobGraph(direction,localValue,neighbor) = 1; // Set the BlobGraph to TRUE for this pair
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void Domain::InitializeRanks()
|
||||
{
|
||||
int i,j,k;
|
||||
// map the rank to the block index
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocx*jproc;
|
||||
@ -258,6 +197,11 @@ void Domain::CommInit(MPI_Comm Communicator){
|
||||
//......................................................................................
|
||||
//Get the ranks of each process and it's neighbors
|
||||
// map the rank to the block index
|
||||
//iproc = rank%nprocx;
|
||||
//jproc = (rank/nprocx)%nprocy;
|
||||
//kproc = rank/(nprocx*nprocy);
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocx*jproc;
|
||||
@ -329,6 +273,7 @@ void Domain::CommInit(MPI_Comm Communicator){
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// allocate send lists
|
||||
sendList_x = new int [sendCount_x];
|
||||
sendList_y = new int [sendCount_y];
|
||||
@ -384,6 +329,7 @@ void Domain::CommInit(MPI_Comm Communicator){
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// allocate send buffers
|
||||
sendBuf_x = new int [sendCount_x];
|
||||
sendBuf_y = new int [sendCount_y];
|
||||
@ -585,6 +531,618 @@ void Domain::CommInit(MPI_Comm Communicator){
|
||||
|
||||
}
|
||||
|
||||
|
||||
void Domain::AssignComponentLabels(double *phase)
|
||||
{
|
||||
int NLABELS=0;
|
||||
char VALUE=0;
|
||||
double AFFINITY=0.f;
|
||||
|
||||
vector <char> Label;
|
||||
vector <double> Affinity;
|
||||
// Read the labels
|
||||
// if (rank==0){
|
||||
/* printf("Component labels:\n");
|
||||
ifstream iFILE("ComponentLabels.csv");
|
||||
if (iFILE.good()){
|
||||
while (!iFILE.eof()){
|
||||
iFILE>>VALUE;
|
||||
iFILE>>AFFINITY;
|
||||
Label.push_back(VALUE);
|
||||
Affinity.push_back(AFFINITY);
|
||||
NLABELS++;
|
||||
printf("%i %f\n",VALUE,AFFINITY);
|
||||
}
|
||||
}
|
||||
else{
|
||||
*/
|
||||
if (rank ==0 ) {
|
||||
printf("Using default labels: Solid (0 --> -1.0), NWP (1 --> 1.0), WP (2 --> -1.0)\n");
|
||||
}
|
||||
// Set default values
|
||||
VALUE=0; AFFINITY=-1.0;
|
||||
Label.push_back(VALUE);
|
||||
Affinity.push_back(AFFINITY);
|
||||
NLABELS++;
|
||||
VALUE=1; AFFINITY=1.0;
|
||||
Label.push_back(VALUE);
|
||||
Affinity.push_back(AFFINITY);
|
||||
NLABELS++;
|
||||
VALUE=2; AFFINITY=-1.0;
|
||||
Label.push_back(VALUE);
|
||||
Affinity.push_back(AFFINITY);
|
||||
NLABELS++;
|
||||
// }
|
||||
// }
|
||||
// Broadcast the list
|
||||
// MPI_Bcast(&NLABELS,1,MPI_INT,0,Comm);
|
||||
|
||||
// Copy into contiguous buffers
|
||||
//char *LabelList;
|
||||
//double * AffinityList;
|
||||
//LabelList=new char[NLABELS];
|
||||
//AffinityList=new double[NLABELS];
|
||||
//MPI_Bcast(&LabelList,NLABELS,MPI_CHAR,0,Comm);
|
||||
//MPI_Bcast(&AffinityList,NLABELS,MPI_DOUBLE,0,Comm);
|
||||
|
||||
// Assign the labels
|
||||
for (int k=0;k<Nz;k++){
|
||||
for (int j=0;j<Ny;j++){
|
||||
for (int i=0;i<Nx;i++){
|
||||
int n = k*Nx*Ny+j*Nx+i;
|
||||
VALUE=id[n];
|
||||
// Assign the affinity from the paired list
|
||||
for (int idx=0; idx < NLABELS; idx++){
|
||||
if (VALUE == Label[idx]){
|
||||
AFFINITY=Affinity[idx];
|
||||
idx = NLABELS;
|
||||
}
|
||||
}
|
||||
phase[n] = AFFINITY;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void Domain::TestCommInit(MPI_Comm Communicator){
|
||||
int i,j,k,n;
|
||||
int sendtag = 21;
|
||||
int recvtag = 21;
|
||||
|
||||
//......................................................................................
|
||||
//Get the ranks of each process and it's neighbors
|
||||
// map the rank to the block index
|
||||
iproc = rank%nprocx;
|
||||
jproc = (rank/nprocx)%nprocy;
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
// set up the neighbor ranks
|
||||
i = iproc;
|
||||
j = jproc;
|
||||
k = kproc;
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
|
||||
if (rank == 0) {
|
||||
printf("* In Domain::CommInit...\n");
|
||||
printf("* i,j,k proc=%d %d %d \n",i,j,k);
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
if (rank == 1){
|
||||
printf("* i,j,k proc=%d %d %d \n",i,j,k);
|
||||
printf("\n\n");
|
||||
}
|
||||
|
||||
|
||||
|
||||
if(rank == 0) { printf("* Setting up ranks for each processor...\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
rank_X = getRankForBlock(i+1,j,k);
|
||||
rank_x = getRankForBlock(i-1,j,k);
|
||||
|
||||
|
||||
if (rank ==0 ) printf("rank = %d: rank_X = %d, rank_x = %d \n", rank, rank_X, rank_x);
|
||||
if (rank ==1 ) printf("rank = %d: rank_X = %d, rank_x = %d \n", rank, rank_X, rank_x);
|
||||
if (rank ==2 ) printf("rank = %d: rank_X = %d, rank_x = %d \n", rank, rank_X, rank_x);
|
||||
|
||||
rank_Y = getRankForBlock(i,j+1,k);
|
||||
rank_y = getRankForBlock(i,j-1,k);
|
||||
rank_Z = getRankForBlock(i,j,k+1);
|
||||
rank_z = getRankForBlock(i,j,k-1);
|
||||
rank_XY = getRankForBlock(i+1,j+1,k);
|
||||
rank_xy = getRankForBlock(i-1,j-1,k);
|
||||
rank_Xy = getRankForBlock(i+1,j-1,k);
|
||||
rank_xY = getRankForBlock(i-1,j+1,k);
|
||||
rank_XZ = getRankForBlock(i+1,j,k+1);
|
||||
rank_xz = getRankForBlock(i-1,j,k-1);
|
||||
rank_Xz = getRankForBlock(i+1,j,k-1);
|
||||
rank_xZ = getRankForBlock(i-1,j,k+1);
|
||||
rank_YZ = getRankForBlock(i,j+1,k+1);
|
||||
rank_yz = getRankForBlock(i,j-1,k-1);
|
||||
rank_Yz = getRankForBlock(i,j+1,k-1);
|
||||
rank_yZ = getRankForBlock(i,j-1,k+1);
|
||||
//......................................................................................
|
||||
|
||||
MPI_Comm_group(Communicator,&Group);
|
||||
MPI_Comm_create(Communicator,Group,&Comm);
|
||||
|
||||
//......................................................................................
|
||||
MPI_Request req1[18], req2[18];
|
||||
MPI_Status stat1[18],stat2[18];
|
||||
//......................................................................................
|
||||
|
||||
for (k=1; k<Nz-1; k++){
|
||||
for (j=1; j<Ny-1; j++){
|
||||
for (i=1; i<Nx-1; i++){
|
||||
// Check the phase ID
|
||||
if (id[k*Nx*Ny+j*Nx+i] != 0){
|
||||
// Counts for the six faces
|
||||
if (i==1) sendCount_x++;
|
||||
if (j==1) sendCount_y++;
|
||||
if (k==1) sendCount_z++;
|
||||
if (i==Nx-2) sendCount_X++;
|
||||
if (j==Ny-2) sendCount_Y++;
|
||||
if (k==Nz-2) sendCount_Z++;
|
||||
// Counts for the twelve edges
|
||||
if (i==1 && j==1) sendCount_xy++;
|
||||
if (i==1 && j==Ny-2) sendCount_xY++;
|
||||
if (i==Nx-2 && j==1) sendCount_Xy++;
|
||||
if (i==Nx-2 && j==Ny-2) sendCount_XY++;
|
||||
|
||||
if (i==1 && k==1) sendCount_xz++;
|
||||
if (i==1 && k==Nz-2) sendCount_xZ++;
|
||||
if (i==Nx-2 && k==1) sendCount_Xz++;
|
||||
if (i==Nx-2 && k==Nz-2) sendCount_XZ++;
|
||||
|
||||
if (j==1 && k==1) sendCount_yz++;
|
||||
if (j==1 && k==Nz-2) sendCount_yZ++;
|
||||
if (j==Ny-2 && k==1) sendCount_Yz++;
|
||||
if (j==Ny-2 && k==Nz-2) sendCount_YZ++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rank == 0) {
|
||||
printf("* All sendCount_# should be the same across multiple processors for block-type domains except for when solid is in the domain... *\n\n");
|
||||
printf("* sendCount_x: %d \n",sendCount_x);
|
||||
printf("* sendCount_y: %d \n",sendCount_y);
|
||||
printf("* sendCount_z: %d \n",sendCount_z);
|
||||
printf("* sendCount_X: %d \n",sendCount_X);
|
||||
printf("* sendCount_Y: %d \n",sendCount_Y);
|
||||
printf("* sendCount_Z: %d \n",sendCount_Z);
|
||||
printf("* sendCount_xy:%d \n",sendCount_xy);
|
||||
printf("* sendCount_xY:%d \n",sendCount_xY);
|
||||
printf("* sendCount_Xy:%d \n",sendCount_Xy);
|
||||
printf("* sendCount_XY:%d \n",sendCount_XY);
|
||||
printf("* sendCount_xz:%d \n",sendCount_xz);
|
||||
printf("* sendCount_xZ:%d \n",sendCount_xZ);
|
||||
printf("* sendCount_Xz:%d \n",sendCount_Xz);
|
||||
printf("* sendCount_XZ:%d \n",sendCount_XZ);
|
||||
printf("* sendCount_yz:%d \n",sendCount_yz);
|
||||
printf("* sendCount_yZ:%d \n",sendCount_yZ);
|
||||
printf("* sendCount_Yz:%d \n",sendCount_Yz);
|
||||
printf("* sendCount_YZ:%d \n",sendCount_YZ);
|
||||
printf("\n\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
if (rank == 1) {
|
||||
printf("* sendCount_x: %d \n",sendCount_x);
|
||||
printf("* sendCount_y: %d \n",sendCount_y);
|
||||
printf("* sendCount_z: %d \n",sendCount_z);
|
||||
printf("* sendCount_X: %d \n",sendCount_X);
|
||||
printf("* sendCount_Y: %d \n",sendCount_Y);
|
||||
printf("* sendCount_Z: %d \n",sendCount_Z);
|
||||
printf("* sendCount_xy:%d \n",sendCount_xy);
|
||||
printf("* sendCount_xY:%d \n",sendCount_xY);
|
||||
printf("* sendCount_Xy:%d \n",sendCount_Xy);
|
||||
printf("* sendCount_XY:%d \n",sendCount_XY);
|
||||
printf("* sendCount_xz:%d \n",sendCount_xz);
|
||||
printf("* sendCount_xZ:%d \n",sendCount_xZ);
|
||||
printf("* sendCount_Xz:%d \n",sendCount_Xz);
|
||||
printf("* sendCount_XZ:%d \n",sendCount_XZ);
|
||||
printf("* sendCount_yz:%d \n",sendCount_yz);
|
||||
printf("* sendCount_yZ:%d \n",sendCount_yZ);
|
||||
printf("* sendCount_Yz:%d \n",sendCount_Yz);
|
||||
printf("* sendCount_YZ:%d \n",sendCount_YZ);
|
||||
printf("\n\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
if (rank == 0) printf("* sendList_# has been allocated through construction of Dm but sizes not determined. Creating arrays with size sendCount_#... *\n");
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
// allocate send lists
|
||||
sendList_x = new int [sendCount_x];
|
||||
sendList_y = new int [sendCount_y];
|
||||
sendList_z = new int [sendCount_z];
|
||||
sendList_X = new int [sendCount_X];
|
||||
sendList_Y = new int [sendCount_Y];
|
||||
sendList_Z = new int [sendCount_Z];
|
||||
sendList_xy = new int [sendCount_xy];
|
||||
sendList_yz = new int [sendCount_yz];
|
||||
sendList_xz = new int [sendCount_xz];
|
||||
sendList_Xy = new int [sendCount_Xy];
|
||||
sendList_Yz = new int [sendCount_Yz];
|
||||
sendList_xZ = new int [sendCount_xZ];
|
||||
sendList_xY = new int [sendCount_xY];
|
||||
sendList_yZ = new int [sendCount_yZ];
|
||||
sendList_Xz = new int [sendCount_Xz];
|
||||
sendList_XY = new int [sendCount_XY];
|
||||
sendList_YZ = new int [sendCount_YZ];
|
||||
sendList_XZ = new int [sendCount_XZ];
|
||||
|
||||
// Populate the send list
|
||||
sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0;
|
||||
sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0;
|
||||
sendCount_xY = sendCount_yZ = sendCount_Xz = sendCount_XY = sendCount_YZ = sendCount_XZ = 0;
|
||||
|
||||
for (k=1; k<Nz-1; k++){
|
||||
for (j=1; j<Ny-1; j++){
|
||||
for (i=1; i<Nx-1; i++){
|
||||
// Local value to send
|
||||
n = k*Nx*Ny+j*Nx+i;
|
||||
if (id[n] != 0){
|
||||
// Counts for the six faces
|
||||
if (i==1) sendList_x[sendCount_x++]=n;
|
||||
if (j==1) sendList_y[sendCount_y++]=n;
|
||||
if (k==1) sendList_z[sendCount_z++]=n;
|
||||
if (i==Nx-2) sendList_X[sendCount_X++]=n;
|
||||
if (j==Ny-2) sendList_Y[sendCount_Y++]=n;
|
||||
if (k==Nz-2) sendList_Z[sendCount_Z++]=n;
|
||||
// Counts for the twelve edges
|
||||
if (i==1 && j==1) sendList_xy[sendCount_xy++]=n;
|
||||
if (i==1 && j==Ny-2) sendList_xY[sendCount_xY++]=n;
|
||||
if (i==Nx-2 && j==1) sendList_Xy[sendCount_Xy++]=n;
|
||||
if (i==Nx-2 && j==Ny-2) sendList_XY[sendCount_XY++]=n;
|
||||
|
||||
if (i==1 && k==1) sendList_xz[sendCount_xz++]=n;
|
||||
if (i==1 && k==Nz-2) sendList_xZ[sendCount_xZ++]=n;
|
||||
if (i==Nx-2 && k==1) sendList_Xz[sendCount_Xz++]=n;
|
||||
if (i==Nx-2 && k==Nz-2) sendList_XZ[sendCount_XZ++]=n;
|
||||
|
||||
if (j==1 && k==1) sendList_yz[sendCount_yz++]=n;
|
||||
if (j==1 && k==Nz-2) sendList_yZ[sendCount_yZ++]=n;
|
||||
if (j==Ny-2 && k==1) sendList_Yz[sendCount_Yz++]=n;
|
||||
if (j==Ny-2 && k==Nz-2) sendList_YZ[sendCount_YZ++]=n;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (rank == 0) {
|
||||
printf("* All blocks should be sending information to the same locations relative to its block - so the data should be the same across processors... *\n");
|
||||
printf("* Expecting some random memory addresses... \n\n");
|
||||
printf("* sendList_x: %d %d %d %d \n",sendList_x[0],sendList_x[1],sendList_x[2],sendList_x[3]);
|
||||
printf("* sendList_y: %d %d %d %d \n",sendList_y[0],sendList_y[1],sendList_y[2],sendList_y[3]);
|
||||
printf("* sendList_z: %d %d %d %d \n",sendList_z[0],sendList_z[1],sendList_z[2],sendList_z[3]);
|
||||
printf("* sendList_X: %d %d %d %d \n",sendList_X[0],sendList_X[1],sendList_X[2],sendList_X[3]);
|
||||
printf("* sendList_Y: %d %d %d %d \n",sendList_Y[0],sendList_Y[1],sendList_Y[2],sendList_Y[3]);
|
||||
printf("* sendList_Z: %d %d %d %d \n",sendList_Z[0],sendList_Z[1],sendList_Z[2],sendList_Z[3]);
|
||||
printf("* sendList_xy:%d %d %d %d \n",sendList_xy[0],sendList_xy[1],sendList_xy[2],sendList_xy[3]);
|
||||
printf("* sendList_xY:%d %d %d %d \n",sendList_xY[0],sendList_xY[1],sendList_xY[2],sendList_xY[3]);
|
||||
printf("* sendList_Xy:%d %d %d %d \n",sendList_Xy[0],sendList_Xy[1],sendList_Xy[2],sendList_Xy[3]);
|
||||
printf("* sendList_XY:%d %d %d %d \n",sendList_XY[0],sendList_XY[1],sendList_XY[2],sendList_XY[3]);
|
||||
printf("* sendList_xz:%d %d %d %d \n",sendList_xz[0],sendList_xz[1],sendList_xz[2],sendList_xz[3]);
|
||||
printf("* sendList_xZ:%d %d %d %d \n",sendList_xZ[0],sendList_xZ[1],sendList_xZ[2],sendList_xZ[3]);
|
||||
printf("* sendList_Xz:%d %d %d %d \n",sendList_Xz[0],sendList_Xz[1],sendList_Xz[2],sendList_Xz[3]);
|
||||
printf("* sendList_XZ:%d %d %d %d \n",sendList_XZ[0],sendList_XZ[1],sendList_XZ[2],sendList_XZ[3]);
|
||||
printf("* sendList_yz:%d %d %d %d \n",sendList_yz[0],sendList_yz[1],sendList_yz[2],sendList_yz[3]);
|
||||
printf("* sendList_yZ:%d %d %d %d \n",sendList_yZ[0],sendList_yZ[1],sendList_yZ[2],sendList_yZ[3]);
|
||||
printf("* sendList_Yz:%d %d %d %d \n",sendList_Yz[0],sendList_Yz[1],sendList_Yz[2],sendList_Yz[3]);
|
||||
printf("* sendList_YZ:%d %d %d %d \n",sendList_YZ[0],sendList_YZ[1],sendList_YZ[2],sendList_YZ[3]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
if (rank == 1) {
|
||||
printf("* Expecting some random memory addresses... \n\n");
|
||||
printf("* sendList_x: %d %d %d %d \n",sendList_x[0],sendList_x[1],sendList_x[2],sendList_x[3]);
|
||||
printf("* sendList_y: %d %d %d %d \n",sendList_y[0],sendList_y[1],sendList_y[2],sendList_y[3]);
|
||||
printf("* sendList_z: %d %d %d %d \n",sendList_z[0],sendList_z[1],sendList_z[2],sendList_z[3]);
|
||||
printf("* sendList_X: %d %d %d %d \n",sendList_X[0],sendList_X[1],sendList_X[2],sendList_X[3]);
|
||||
printf("* sendList_Y: %d %d %d %d \n",sendList_Y[0],sendList_Y[1],sendList_Y[2],sendList_Y[3]);
|
||||
printf("* sendList_Z: %d %d %d %d \n",sendList_Z[0],sendList_Z[1],sendList_Z[2],sendList_Z[3]);
|
||||
printf("* sendList_xy:%d %d %d %d \n",sendList_xy[0],sendList_xy[1],sendList_xy[2],sendList_xy[3]);
|
||||
printf("* sendList_xY:%d %d %d %d \n",sendList_xY[0],sendList_xY[1],sendList_xY[2],sendList_xY[3]);
|
||||
printf("* sendList_Xy:%d %d %d %d \n",sendList_Xy[0],sendList_Xy[1],sendList_Xy[2],sendList_Xy[3]);
|
||||
printf("* sendList_XY:%d %d %d %d \n",sendList_XY[0],sendList_XY[1],sendList_XY[2],sendList_XY[3]);
|
||||
printf("* sendList_xz:%d %d %d %d \n",sendList_xz[0],sendList_xz[1],sendList_xz[2],sendList_xz[3]);
|
||||
printf("* sendList_xZ:%d %d %d %d \n",sendList_xZ[0],sendList_xZ[1],sendList_xZ[2],sendList_xZ[3]);
|
||||
printf("* sendList_Xz:%d %d %d %d \n",sendList_Xz[0],sendList_Xz[1],sendList_Xz[2],sendList_Xz[3]);
|
||||
printf("* sendList_XZ:%d %d %d %d \n",sendList_XZ[0],sendList_XZ[1],sendList_XZ[2],sendList_XZ[3]);
|
||||
printf("* sendList_yz:%d %d %d %d \n",sendList_yz[0],sendList_yz[1],sendList_yz[2],sendList_yz[3]);
|
||||
printf("* sendList_yZ:%d %d %d %d \n",sendList_yZ[0],sendList_yZ[1],sendList_yZ[2],sendList_yZ[3]);
|
||||
printf("* sendList_Yz:%d %d %d %d \n",sendList_Yz[0],sendList_Yz[1],sendList_Yz[2],sendList_Yz[3]);
|
||||
printf("* sendList_YZ:%d %d %d %d \n",sendList_YZ[0],sendList_YZ[1],sendList_YZ[2],sendList_YZ[3]);
|
||||
printf("\n\n");
|
||||
}
|
||||
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
if (rank == 0) printf("* sendBuf_# has been allocated through construction of Dm but sizes not determined. Creating arrays with size sendCount_#... *\n");
|
||||
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
|
||||
// allocate send buffers
|
||||
sendBuf_x = new int [sendCount_x];
|
||||
sendBuf_y = new int [sendCount_y];
|
||||
sendBuf_z = new int [sendCount_z];
|
||||
sendBuf_X = new int [sendCount_X];
|
||||
sendBuf_Y = new int [sendCount_Y];
|
||||
sendBuf_Z = new int [sendCount_Z];
|
||||
sendBuf_xy = new int [sendCount_xy];
|
||||
sendBuf_yz = new int [sendCount_yz];
|
||||
sendBuf_xz = new int [sendCount_xz];
|
||||
sendBuf_Xy = new int [sendCount_Xy];
|
||||
sendBuf_Yz = new int [sendCount_Yz];
|
||||
sendBuf_xZ = new int [sendCount_xZ];
|
||||
sendBuf_xY = new int [sendCount_xY];
|
||||
sendBuf_yZ = new int [sendCount_yZ];
|
||||
sendBuf_Xz = new int [sendCount_Xz];
|
||||
sendBuf_XY = new int [sendCount_XY];
|
||||
sendBuf_YZ = new int [sendCount_YZ];
|
||||
sendBuf_XZ = new int [sendCount_XZ];
|
||||
//......................................................................................
|
||||
|
||||
if (rank == 0) printf("* >> Isending and Ireceiving count data across processors...\n");
|
||||
|
||||
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]);
|
||||
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]);
|
||||
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]);
|
||||
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]);
|
||||
|
||||
|
||||
|
||||
|
||||
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]);
|
||||
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]);
|
||||
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]);
|
||||
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]);
|
||||
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]);
|
||||
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]);
|
||||
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]);
|
||||
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]);
|
||||
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]);
|
||||
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]);
|
||||
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]);
|
||||
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]);
|
||||
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]);
|
||||
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]);
|
||||
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]);
|
||||
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]);
|
||||
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]);
|
||||
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]);
|
||||
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]);
|
||||
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]);
|
||||
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]);
|
||||
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]);
|
||||
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]);
|
||||
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]);
|
||||
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]);
|
||||
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]);
|
||||
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]);
|
||||
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]);
|
||||
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]);
|
||||
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]);
|
||||
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]);
|
||||
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(Communicator);
|
||||
//......................................................................................
|
||||
|
||||
if (rank == 0) printf("* recvList_# has been allocated through construction of Dm but sizes not determined. Creating arrays with size recvCount_#... *\n");
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
// recv buffers
|
||||
recvList_x = new int [recvCount_x];
|
||||
recvList_y = new int [recvCount_y];
|
||||
recvList_z = new int [recvCount_z];
|
||||
recvList_X = new int [recvCount_X];
|
||||
recvList_Y = new int [recvCount_Y];
|
||||
recvList_Z = new int [recvCount_Z];
|
||||
recvList_xy = new int [recvCount_xy];
|
||||
recvList_yz = new int [recvCount_yz];
|
||||
recvList_xz = new int [recvCount_xz];
|
||||
recvList_Xy = new int [recvCount_Xy];
|
||||
recvList_Yz = new int [recvCount_Yz];
|
||||
recvList_xZ = new int [recvCount_xZ];
|
||||
recvList_xY = new int [recvCount_xY];
|
||||
recvList_yZ = new int [recvCount_yZ];
|
||||
recvList_Xz = new int [recvCount_Xz];
|
||||
recvList_XY = new int [recvCount_XY];
|
||||
recvList_YZ = new int [recvCount_YZ];
|
||||
recvList_XZ = new int [recvCount_XZ];
|
||||
//......................................................................................
|
||||
|
||||
if (rank == 0) printf("* >> Isending and Ireceiving list data (of size sendCount_# and recvCount_#) across processors...\n");
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]);
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]);
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]);
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
|
||||
|
||||
if (rank == 0) {
|
||||
|
||||
printf("* recvList_x: %d %d %d %d \n",recvList_x[0],recvList_x[1],recvList_x[2],recvList_x[3]);
|
||||
|
||||
printf("* recvList_X: %d %d %d %d \n",recvList_X[0],recvList_X[1],recvList_X[2],recvList_X[3]);
|
||||
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
if (rank == 1) {
|
||||
|
||||
printf("* recvList_x: %d %d %d %d \n",recvList_x[0],recvList_x[1],recvList_x[2],recvList_x[3]);
|
||||
|
||||
printf("* recvList_X: %d %d %d %d \n",recvList_X[0],recvList_X[1],recvList_X[2],recvList_X[3]);
|
||||
|
||||
printf("\n\n");
|
||||
}
|
||||
|
||||
|
||||
|
||||
//......................................................................................
|
||||
for (int idx=0; idx<recvCount_x; idx++) {recvList_x[idx] -= (Nx-2); if (rank ==1) printf("%d ",recvList_x[idx]); }
|
||||
if (rank == 1) printf("\n");
|
||||
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
|
||||
for (int idx=0; idx<recvCount_y; idx++) recvList_y[idx] -= (Ny-2)*Nx;
|
||||
for (int idx=0; idx<recvCount_Y; idx++) recvList_Y[idx] += (Ny-2)*Nx;
|
||||
for (int idx=0; idx<recvCount_z; idx++) recvList_z[idx] -= (Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_Z; idx++) recvList_Z[idx] += (Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_xy; idx++) recvList_xy[idx] -= (Nx-2)+(Ny-2)*Nx;
|
||||
for (int idx=0; idx<recvCount_XY; idx++) recvList_XY[idx] += (Nx-2)+(Ny-2)*Nx;
|
||||
for (int idx=0; idx<recvCount_xY; idx++) recvList_xY[idx] -= (Nx-2)-(Ny-2)*Nx;
|
||||
for (int idx=0; idx<recvCount_Xy; idx++) recvList_Xy[idx] += (Nx-2)-(Ny-2)*Nx;
|
||||
for (int idx=0; idx<recvCount_xz; idx++) recvList_xz[idx] -= (Nx-2)+(Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_XZ; idx++) recvList_XZ[idx] += (Nx-2)+(Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_xZ; idx++) recvList_xZ[idx] -= (Nx-2)-(Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_Xz; idx++) recvList_Xz[idx] += (Nx-2)-(Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_yz; idx++) recvList_yz[idx] -= (Ny-2)*Nx + (Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_YZ; idx++) recvList_YZ[idx] += (Ny-2)*Nx + (Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_yZ; idx++) recvList_yZ[idx] -= (Ny-2)*Nx - (Nz-2)*Nx*Ny;
|
||||
for (int idx=0; idx<recvCount_Yz; idx++) recvList_Yz[idx] += (Ny-2)*Nx - (Nz-2)*Nx*Ny;
|
||||
//......................................................................................
|
||||
|
||||
|
||||
if (rank == 0) printf("* recvBuf_# has been allocated through construction of Dm but sizes not determined. Creating arrays with size recvCount_#... *\n");
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
// allocate recv buffers
|
||||
recvBuf_x = new int [recvCount_x];
|
||||
recvBuf_y = new int [recvCount_y];
|
||||
recvBuf_z = new int [recvCount_z];
|
||||
recvBuf_X = new int [recvCount_X];
|
||||
recvBuf_Y = new int [recvCount_Y];
|
||||
recvBuf_Z = new int [recvCount_Z];
|
||||
recvBuf_xy = new int [recvCount_xy];
|
||||
recvBuf_yz = new int [recvCount_yz];
|
||||
recvBuf_xz = new int [recvCount_xz];
|
||||
recvBuf_Xy = new int [recvCount_Xy];
|
||||
recvBuf_Yz = new int [recvCount_Yz];
|
||||
recvBuf_xZ = new int [recvCount_xZ];
|
||||
recvBuf_xY = new int [recvCount_xY];
|
||||
recvBuf_yZ = new int [recvCount_yZ];
|
||||
recvBuf_Xz = new int [recvCount_Xz];
|
||||
recvBuf_XY = new int [recvCount_XY];
|
||||
recvBuf_YZ = new int [recvCount_YZ];
|
||||
recvBuf_XZ = new int [recvCount_XZ];
|
||||
//......................................................................................
|
||||
|
||||
if (rank == 0) printf("* sendData_# has been allocated through construction of Dm but sizes not determined. Creating arrays with size sendCount_#... *\n");
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
// send buffers
|
||||
sendData_x = new double [sendCount_x];
|
||||
sendData_y = new double [sendCount_y];
|
||||
sendData_z = new double [sendCount_z];
|
||||
sendData_X = new double [sendCount_X];
|
||||
sendData_Y = new double [sendCount_Y];
|
||||
sendData_Z = new double [sendCount_Z];
|
||||
sendData_xy = new double [sendCount_xy];
|
||||
sendData_yz = new double [sendCount_yz];
|
||||
sendData_xz = new double [sendCount_xz];
|
||||
sendData_Xy = new double [sendCount_Xy];
|
||||
sendData_Yz = new double [sendCount_Yz];
|
||||
sendData_xZ = new double [sendCount_xZ];
|
||||
sendData_xY = new double [sendCount_xY];
|
||||
sendData_yZ = new double [sendCount_yZ];
|
||||
sendData_Xz = new double [sendCount_Xz];
|
||||
sendData_XY = new double [sendCount_XY];
|
||||
sendData_YZ = new double [sendCount_YZ];
|
||||
sendData_XZ = new double [sendCount_XZ];
|
||||
//......................................................................................
|
||||
|
||||
if (rank == 0) printf("* recvData_# has been allocated through construction of Dm but sizes not determined. Creating arrays with size recvCount_#... *\n");
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
// recv buffers
|
||||
recvData_x = new double [recvCount_x];
|
||||
recvData_y = new double [recvCount_y];
|
||||
recvData_z = new double [recvCount_z];
|
||||
recvData_X = new double [recvCount_X];
|
||||
recvData_Y = new double [recvCount_Y];
|
||||
recvData_Z = new double [recvCount_Z];
|
||||
recvData_xy = new double [recvCount_xy];
|
||||
recvData_yz = new double [recvCount_yz];
|
||||
recvData_xz = new double [recvCount_xz];
|
||||
recvData_Xy = new double [recvCount_Xy];
|
||||
recvData_xZ = new double [recvCount_xZ];
|
||||
recvData_xY = new double [recvCount_xY];
|
||||
recvData_yZ = new double [recvCount_yZ];
|
||||
recvData_Yz = new double [recvCount_Yz];
|
||||
recvData_Xz = new double [recvCount_Xz];
|
||||
recvData_XY = new double [recvCount_XY];
|
||||
recvData_YZ = new double [recvCount_YZ];
|
||||
recvData_XZ = new double [recvCount_XZ];
|
||||
//......................................................................................
|
||||
|
||||
if (rank == 0) printf("* End of TestCommInit...\n\n");
|
||||
}
|
||||
|
||||
void Domain::CommunicateMeshHalo(DoubleArray &Mesh)
|
||||
{
|
||||
int sendtag, recvtag;
|
||||
@ -666,89 +1224,3 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh)
|
||||
UnpackMeshData(recvList_YZ, recvCount_YZ ,recvData_YZ, MeshData);
|
||||
}
|
||||
|
||||
|
||||
void Domain::BlobComm(MPI_Comm Communicator)
|
||||
{
|
||||
//......................................................................................
|
||||
int sendtag, recvtag;
|
||||
sendtag = recvtag = 51;
|
||||
//......................................................................................
|
||||
int *BlobLabelData = BlobLabel.data();
|
||||
PackBlobData(sendList_x, sendCount_x ,sendBuf_x, BlobLabelData);
|
||||
PackBlobData(sendList_X, sendCount_X ,sendBuf_X, BlobLabelData);
|
||||
PackBlobData(sendList_y, sendCount_y ,sendBuf_y, BlobLabelData);
|
||||
PackBlobData(sendList_Y, sendCount_Y ,sendBuf_Y, BlobLabelData);
|
||||
PackBlobData(sendList_z, sendCount_z ,sendBuf_z, BlobLabelData);
|
||||
PackBlobData(sendList_Z, sendCount_Z ,sendBuf_Z, BlobLabelData);
|
||||
PackBlobData(sendList_xy, sendCount_xy ,sendBuf_xy, BlobLabelData);
|
||||
PackBlobData(sendList_Xy, sendCount_Xy ,sendBuf_Xy, BlobLabelData);
|
||||
PackBlobData(sendList_xY, sendCount_xY ,sendBuf_xY, BlobLabelData);
|
||||
PackBlobData(sendList_XY, sendCount_XY ,sendBuf_XY, BlobLabelData);
|
||||
PackBlobData(sendList_xz, sendCount_xz ,sendBuf_xz, BlobLabelData);
|
||||
PackBlobData(sendList_Xz, sendCount_Xz ,sendBuf_Xz, BlobLabelData);
|
||||
PackBlobData(sendList_xZ, sendCount_xZ ,sendBuf_xZ, BlobLabelData);
|
||||
PackBlobData(sendList_XZ, sendCount_XZ ,sendBuf_XZ, BlobLabelData);
|
||||
PackBlobData(sendList_yz, sendCount_yz ,sendBuf_yz, BlobLabelData);
|
||||
PackBlobData(sendList_Yz, sendCount_Yz ,sendBuf_Yz, BlobLabelData);
|
||||
PackBlobData(sendList_yZ, sendCount_yZ ,sendBuf_yZ, BlobLabelData);
|
||||
PackBlobData(sendList_YZ, sendCount_YZ ,sendBuf_YZ, BlobLabelData);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendBuf_x,sendCount_x,MPI_INT,rank_x,sendtag,
|
||||
recvBuf_X,recvCount_X,MPI_INT,rank_X,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_X,sendCount_X,MPI_INT,rank_X,sendtag,
|
||||
recvBuf_x,recvCount_x,MPI_INT,rank_x,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_y,sendCount_y,MPI_INT,rank_y,sendtag,
|
||||
recvBuf_Y,recvCount_Y,MPI_INT,rank_Y,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_Y,sendCount_Y,MPI_INT,rank_Y,sendtag,
|
||||
recvBuf_y,recvCount_y,MPI_INT,rank_y,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_z,sendCount_z,MPI_INT,rank_z,sendtag,
|
||||
recvBuf_Z,recvCount_Z,MPI_INT,rank_Z,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_Z,sendCount_Z,MPI_INT,rank_Z,sendtag,
|
||||
recvBuf_z,recvCount_z,MPI_INT,rank_z,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_xy,sendCount_xy,MPI_INT,rank_xy,sendtag,
|
||||
recvBuf_XY,recvCount_XY,MPI_INT,rank_XY,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_XY,sendCount_XY,MPI_INT,rank_XY,sendtag,
|
||||
recvBuf_xy,recvCount_xy,MPI_INT,rank_xy,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_Xy,sendCount_Xy,MPI_INT,rank_Xy,sendtag,
|
||||
recvBuf_xY,recvCount_xY,MPI_INT,rank_xY,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_xY,sendCount_xY,MPI_INT,rank_xY,sendtag,
|
||||
recvBuf_Xy,recvCount_Xy,MPI_INT,rank_Xy,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_xz,sendCount_xz,MPI_INT,rank_xz,sendtag,
|
||||
recvBuf_XZ,recvCount_XZ,MPI_INT,rank_XZ,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_XZ,sendCount_XZ,MPI_INT,rank_XZ,sendtag,
|
||||
recvBuf_xz,recvCount_xz,MPI_INT,rank_xz,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_Xz,sendCount_Xz,MPI_INT,rank_Xz,sendtag,
|
||||
recvBuf_xZ,recvCount_xZ,MPI_INT,rank_xZ,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_xZ,sendCount_xZ,MPI_INT,rank_xZ,sendtag,
|
||||
recvBuf_Xz,recvCount_Xz,MPI_INT,rank_Xz,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_yz,sendCount_yz,MPI_INT,rank_yz,sendtag,
|
||||
recvBuf_YZ,recvCount_YZ,MPI_INT,rank_YZ,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_YZ,sendCount_YZ,MPI_INT,rank_YZ,sendtag,
|
||||
recvBuf_yz,recvCount_yz,MPI_INT,rank_yz,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_Yz,sendCount_Yz,MPI_INT,rank_Yz,sendtag,
|
||||
recvBuf_yZ,recvCount_yZ,MPI_INT,rank_yZ,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendBuf_yZ,sendCount_yZ,MPI_INT,rank_yZ,sendtag,
|
||||
recvBuf_Yz,recvCount_Yz,MPI_INT,rank_Yz,recvtag,Comm,MPI_STATUS_IGNORE);
|
||||
//........................................................................................
|
||||
UnpackBlobData(recvList_x, recvCount_x ,recvBuf_x, BlobLabelData);
|
||||
UnpackBlobData(recvList_X, recvCount_X ,recvBuf_X, BlobLabelData);
|
||||
UnpackBlobData(recvList_y, recvCount_y ,recvBuf_y, BlobLabelData);
|
||||
UnpackBlobData(recvList_Y, recvCount_Y ,recvBuf_Y, BlobLabelData);
|
||||
UnpackBlobData(recvList_z, recvCount_z ,recvBuf_z, BlobLabelData);
|
||||
UnpackBlobData(recvList_Z, recvCount_Z ,recvBuf_Z, BlobLabelData);
|
||||
UnpackBlobData(recvList_xy, recvCount_xy ,recvBuf_xy, BlobLabelData);
|
||||
UnpackBlobData(recvList_Xy, recvCount_Xy ,recvBuf_Xy, BlobLabelData);
|
||||
UnpackBlobData(recvList_xY, recvCount_xY ,recvBuf_xY, BlobLabelData);
|
||||
UnpackBlobData(recvList_XY, recvCount_XY ,recvBuf_XY, BlobLabelData);
|
||||
UnpackBlobData(recvList_xz, recvCount_xz ,recvBuf_xz, BlobLabelData);
|
||||
UnpackBlobData(recvList_Xz, recvCount_Xz ,recvBuf_Xz, BlobLabelData);
|
||||
UnpackBlobData(recvList_xZ, recvCount_xZ ,recvBuf_xZ, BlobLabelData);
|
||||
UnpackBlobData(recvList_XZ, recvCount_XZ ,recvBuf_XZ, BlobLabelData);
|
||||
UnpackBlobData(recvList_yz, recvCount_yz ,recvBuf_yz, BlobLabelData);
|
||||
UnpackBlobData(recvList_Yz, recvCount_Yz ,recvBuf_Yz, BlobLabelData);
|
||||
UnpackBlobData(recvList_yZ, recvCount_yZ ,recvBuf_yZ, BlobLabelData);
|
||||
UnpackBlobData(recvList_YZ, recvCount_YZ ,recvBuf_YZ, BlobLabelData);
|
||||
//......................................................................................
|
||||
}
|
||||
|
||||
|
||||
|
||||
187
common/Domain.h
187
common/Domain.h
@ -24,6 +24,7 @@ void read_domain( int rank, int nprocs, MPI_Comm comm,
|
||||
int& nprocx, int& nprocy, int& nprocz, int& nx, int& ny, int& nz,
|
||||
int& nspheres, double& Lx, double& Ly, double& Lz );
|
||||
|
||||
|
||||
//! Class to hold domain info
|
||||
struct Domain{
|
||||
// Default constructor
|
||||
@ -89,16 +90,15 @@ struct Domain{
|
||||
|
||||
// Solid indicator function
|
||||
char *id;
|
||||
// Blob information
|
||||
IntArray BlobLabel;
|
||||
IntArray BlobGraph;
|
||||
|
||||
void InitializeRanks();
|
||||
void CommInit(MPI_Comm comm);
|
||||
void CommunicateMeshHalo(DoubleArray &Mesh);
|
||||
void BlobComm(MPI_Comm comm);
|
||||
void AssignComponentLabels(double *phase);
|
||||
|
||||
void AssignBlobConnections();
|
||||
void TestCommInit(MPI_Comm comm);
|
||||
|
||||
//void MemoryOptimizedLayout(IntArray &Map, int *neighborList, int Np);
|
||||
|
||||
private:
|
||||
|
||||
@ -109,28 +109,6 @@ private:
|
||||
int k2 = (k+nprocz)%nprocz;
|
||||
return i2 + j2*nprocx + k2*nprocx*nprocy;
|
||||
}
|
||||
inline void PackBlobData(int *list, int count, int *sendbuf, int *data){
|
||||
// Fill in the phase ID values from neighboring processors
|
||||
// This packs up the values that need to be sent from one processor to another
|
||||
int idx,n;
|
||||
for (idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
sendbuf[idx] = data[n];
|
||||
}
|
||||
}
|
||||
inline void UnpackBlobData(int *list, int count, int *recvbuf, int *data){
|
||||
// Fill in the phase ID values from neighboring processors
|
||||
// This unpacks the values once they have been recieved from neighbors
|
||||
int idx,n;
|
||||
for (idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
data[n] = recvbuf[idx];
|
||||
}
|
||||
}
|
||||
|
||||
int VoxelConnection(int n);
|
||||
|
||||
void getBlobConnections(int * List, int count, int neighbor, int direction);
|
||||
};
|
||||
|
||||
|
||||
@ -596,161 +574,6 @@ inline void ReadBinaryFile(char *FILENAME, double *Data, int N)
|
||||
File.close();
|
||||
|
||||
}
|
||||
inline double minmod(double &a, double &b){
|
||||
|
||||
double value;
|
||||
|
||||
value = a;
|
||||
if ( a*b < 0.0) value=0.0;
|
||||
else if (fabs(a) > fabs(b)) value = b;
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
inline double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){
|
||||
|
||||
/*
|
||||
* This routine converts the data in the Distance array to a signed distance
|
||||
* by solving the equation df/dt = sign(1-|grad f|), where Distance provides
|
||||
* the values of f on the mesh associated with domain Dm
|
||||
* It has been tested with segmented data initialized to values [-1,1]
|
||||
* and will converge toward the signed distance to the surface bounding the associated phases
|
||||
*
|
||||
* Reference:
|
||||
* Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229
|
||||
*/
|
||||
|
||||
int i,j,k;
|
||||
double dt=0.1;
|
||||
double Dx,Dy,Dz;
|
||||
double Dxp,Dxm,Dyp,Dym,Dzp,Dzm;
|
||||
double Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm;
|
||||
double sign,norm;
|
||||
double LocalVar,GlobalVar,LocalMax,GlobalMax;
|
||||
|
||||
int xdim,ydim,zdim;
|
||||
xdim=Dm.Nx-2;
|
||||
ydim=Dm.Ny-2;
|
||||
zdim=Dm.Nz-2;
|
||||
fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
|
||||
|
||||
// Arrays to store the second derivatives
|
||||
DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz);
|
||||
DoubleArray Dyy(Dm.Nx,Dm.Ny,Dm.Nz);
|
||||
DoubleArray Dzz(Dm.Nx,Dm.Ny,Dm.Nz);
|
||||
|
||||
int count = 0;
|
||||
while (count < timesteps){
|
||||
|
||||
// Communicate the halo of values
|
||||
fillData.fill(Distance);
|
||||
|
||||
// Compute second order derivatives
|
||||
for (k=1;k<Dm.Nz-1;k++){
|
||||
for (j=1;j<Dm.Ny-1;j++){
|
||||
for (i=1;i<Dm.Nx-1;i++){
|
||||
Dxx(i,j,k) = Distance(i+1,j,k) + Distance(i-1,j,k) - 2*Distance(i,j,k);
|
||||
Dyy(i,j,k) = Distance(i,j+1,k) + Distance(i,j-1,k) - 2*Distance(i,j,k);
|
||||
Dzz(i,j,k) = Distance(i,j,k+1) + Distance(i,j,k-1) - 2*Distance(i,j,k);
|
||||
}
|
||||
}
|
||||
}
|
||||
fillData.fill(Dxx);
|
||||
fillData.fill(Dyy);
|
||||
fillData.fill(Dzz);
|
||||
|
||||
LocalMax=LocalVar=0.0;
|
||||
// Execute the next timestep
|
||||
for (k=1;k<Dm.Nz-1;k++){
|
||||
for (j=1;j<Dm.Ny-1;j++){
|
||||
for (i=1;i<Dm.Nx-1;i++){
|
||||
|
||||
int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
|
||||
|
||||
sign = 1;
|
||||
if (ID[n] == 0) sign = -1;
|
||||
|
||||
// local second derivative terms
|
||||
Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k));
|
||||
Dyyp = minmod(Dyy(i,j,k),Dyy(i,j+1,k));
|
||||
Dzzp = minmod(Dzz(i,j,k),Dzz(i,j,k+1));
|
||||
Dxxm = minmod(Dxx(i,j,k),Dxx(i-1,j,k));
|
||||
Dyym = minmod(Dyy(i,j,k),Dyy(i,j-1,k));
|
||||
Dzzm = minmod(Dzz(i,j,k),Dzz(i,j,k-1));
|
||||
|
||||
/* //............Compute upwind derivatives ...................
|
||||
Dxp = Distance(i+1,j,k) - Distance(i,j,k) + 0.5*Dxxp;
|
||||
Dyp = Distance(i,j+1,k) - Distance(i,j,k) + 0.5*Dyyp;
|
||||
Dzp = Distance(i,j,k+1) - Distance(i,j,k) + 0.5*Dzzp;
|
||||
|
||||
|
||||
Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
|
||||
Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
|
||||
Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
|
||||
*/
|
||||
Dxp = Distance(i+1,j,k)- Distance(i,j,k) - 0.5*Dxxp;
|
||||
Dyp = Distance(i,j+1,k)- Distance(i,j,k) - 0.5*Dyyp;
|
||||
Dzp = Distance(i,j,k+1)- Distance(i,j,k) - 0.5*Dzzp;
|
||||
|
||||
Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
|
||||
Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
|
||||
Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
|
||||
|
||||
// Compute upwind derivatives for Godunov Hamiltonian
|
||||
if (sign < 0.0){
|
||||
if (Dxp + Dxm > 0.f) Dx = Dxp*Dxp;
|
||||
else Dx = Dxm*Dxm;
|
||||
|
||||
if (Dyp + Dym > 0.f) Dy = Dyp*Dyp;
|
||||
else Dy = Dym*Dym;
|
||||
|
||||
if (Dzp + Dzm > 0.f) Dz = Dzp*Dzp;
|
||||
else Dz = Dzm*Dzm;
|
||||
}
|
||||
else{
|
||||
|
||||
if (Dxp + Dxm < 0.f) Dx = Dxp*Dxp;
|
||||
else Dx = Dxm*Dxm;
|
||||
|
||||
if (Dyp + Dym < 0.f) Dy = Dyp*Dyp;
|
||||
else Dy = Dym*Dym;
|
||||
|
||||
if (Dzp + Dzm < 0.f) Dz = Dzp*Dzp;
|
||||
else Dz = Dzm*Dzm;
|
||||
}
|
||||
|
||||
//Dx = max(Dxp*Dxp,Dxm*Dxm);
|
||||
//Dy = max(Dyp*Dyp,Dym*Dym);
|
||||
//Dz = max(Dzp*Dzp,Dzm*Dzm);
|
||||
|
||||
norm=sqrt(Dx + Dy + Dz);
|
||||
if (norm > 1.0) norm=1.0;
|
||||
|
||||
Distance(i,j,k) += dt*sign*(1.0 - norm);
|
||||
LocalVar += dt*sign*(1.0 - norm);
|
||||
|
||||
if (fabs(dt*sign*(1.0 - norm)) > LocalMax)
|
||||
LocalMax = fabs(dt*sign*(1.0 - norm));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
|
||||
MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm);
|
||||
GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz;
|
||||
count++;
|
||||
|
||||
|
||||
if (count%50 == 0 && Dm.rank==0 )
|
||||
printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar);
|
||||
|
||||
if (fabs(GlobalMax) < 1e-5){
|
||||
if (Dm.rank==0) printf("Exiting with max tolerance of 1e-5 \n");
|
||||
count=timesteps;
|
||||
}
|
||||
}
|
||||
return GlobalVar;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
@ -27,7 +27,7 @@ void pack( const std::vector<TYPE>& rhs, char *buffer )
|
||||
size_t size = rhs.size();
|
||||
memcpy(buffer,&size,sizeof(size_t));
|
||||
size_t pos = sizeof(size_t);
|
||||
for (size_t i=0; i<rhs.size(); i++) {
|
||||
for (int i=0; i<rhs.size(); i++) {
|
||||
pack(rhs[i],&buffer[pos]);
|
||||
pos += packsize(rhs[i]);
|
||||
}
|
||||
@ -40,7 +40,7 @@ void unpack( std::vector<TYPE>& data, const char *buffer )
|
||||
data.clear();
|
||||
data.resize(size);
|
||||
size_t pos = sizeof(size_t);
|
||||
for (size_t i=0; i<data.size(); i++) {
|
||||
for (int i=0; i<data.size(); i++) {
|
||||
unpack(data[i],&buffer[pos]);
|
||||
pos += packsize(data[i]);
|
||||
}
|
||||
|
||||
3407
common/ScaLBL.h
3407
common/ScaLBL.h
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,31 +1,12 @@
|
||||
#ifndef included_AtomicStackTrace
|
||||
#define included_AtomicStackTrace
|
||||
#ifndef included_StackTrace
|
||||
#define included_StackTrace
|
||||
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <thread>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
|
||||
|
||||
// Check for and include MPI
|
||||
// clang-format off
|
||||
#if defined(USE_MPI) || defined(USE_EXT_MPI)
|
||||
#include "mpi.h"
|
||||
#elif defined(__has_include)
|
||||
#if __has_include("mpi.h")
|
||||
#include "mpi.h"
|
||||
#else
|
||||
typedef int MPI_Comm;
|
||||
#endif
|
||||
#else
|
||||
typedef int MPI_Comm;
|
||||
#endif
|
||||
// clang-format on
|
||||
|
||||
|
||||
namespace StackTrace {
|
||||
|
||||
@ -38,179 +19,29 @@ struct stack_info {
|
||||
std::string filename;
|
||||
int line;
|
||||
//! Default constructor
|
||||
stack_info() : address( nullptr ), address2( nullptr ), line( 0 ) {}
|
||||
//! Operator==
|
||||
bool operator==( const stack_info& rhs ) const;
|
||||
//! Operator!=
|
||||
bool operator!=( const stack_info& rhs ) const;
|
||||
stack_info(): address(NULL), address2(NULL), line(0) {}
|
||||
//! Print the stack info
|
||||
std::string print() const;
|
||||
//! Compute the number of bytes needed to store the object
|
||||
size_t size() const;
|
||||
//! Pack the data to a byte array, returning a pointer to the end of the data
|
||||
char* pack( char* ptr ) const;
|
||||
//! Unpack the data from a byte array, returning a pointer to the end of the data
|
||||
const char* unpack( const char* ptr );
|
||||
//! Pack a vector of data to a memory block
|
||||
static std::vector<char> packArray( const std::vector<stack_info>& data );
|
||||
//! Unpack a vector of data from a memory block
|
||||
static std::vector<stack_info> unpackArray( const char* data );
|
||||
};
|
||||
|
||||
|
||||
struct multi_stack_info {
|
||||
int N;
|
||||
stack_info stack;
|
||||
std::vector<multi_stack_info> children;
|
||||
//! Default constructor
|
||||
multi_stack_info() : N( 0 ) {}
|
||||
//! Add the given stack to the multistack
|
||||
void add( size_t N, const stack_info *stack );
|
||||
//! Print the stack info
|
||||
std::vector<std::string> print( const std::string& prefix=std::string() ) const;
|
||||
};
|
||||
|
||||
|
||||
/*!
|
||||
* @brief Get the current call stack
|
||||
* @details This function returns the current call stack for the current thread
|
||||
* @return Returns vector containing the stack
|
||||
*/
|
||||
//! Function to return the current call stack
|
||||
std::vector<stack_info> getCallStack();
|
||||
|
||||
|
||||
/*!
|
||||
* @brief Get the current call stack for a thread
|
||||
* @details This function returns the current call stack for the given thread
|
||||
* @param[in] id The thread id of the stack we want to return
|
||||
* @return Returns vector containing the stack
|
||||
*/
|
||||
std::vector<stack_info> getCallStack( std::thread::native_handle_type id );
|
||||
|
||||
|
||||
/*!
|
||||
* @brief Get the current call stack for all threads
|
||||
* @details This function returns the current call stack for all threads
|
||||
* in the current process.
|
||||
* Note: This functionality may not be availible on all platforms
|
||||
* @return Returns vector containing the stack
|
||||
*/
|
||||
multi_stack_info getAllCallStacks( );
|
||||
|
||||
|
||||
/*!
|
||||
* @brief Get the current call stack for all threads/processes
|
||||
* @details This function returns the current call stack for all threads
|
||||
* for all processes in the current process. This function requires
|
||||
* the user to call globalCallStackInitialize() before calling this
|
||||
* routine, and globalCallStackFinalize() before exiting.
|
||||
* Note: This functionality may not be availible on all platforms
|
||||
* @return Returns vector containing the stack
|
||||
*/
|
||||
multi_stack_info getGlobalCallStacks( );
|
||||
|
||||
|
||||
//! Function to return the current call stack for the current thread
|
||||
std::vector<void *> backtrace();
|
||||
|
||||
//! Function to return the current call stack for the given thread
|
||||
std::vector<void *> backtrace( std::thread::native_handle_type id );
|
||||
|
||||
//! Function to return the current call stack for all threads
|
||||
std::vector<std::vector<void *>> backtraceAll();
|
||||
|
||||
|
||||
//! Function to return the stack info for a given address
|
||||
stack_info getStackInfo( void *address );
|
||||
|
||||
|
||||
//! Function to return the stack info for a given address
|
||||
std::vector<stack_info> getStackInfo( const std::vector<void *> &address );
|
||||
|
||||
|
||||
//! Function to return the signal name
|
||||
std::string signalName( int signal );
|
||||
stack_info getStackInfo( void* address );
|
||||
|
||||
|
||||
/*!
|
||||
* Return the symbols from the current executable (not availible for all platforms)
|
||||
* @return Returns 0 if sucessful
|
||||
*/
|
||||
int getSymbols(
|
||||
std::vector<void *> &address, std::vector<char> &type, std::vector<std::string> &obj );
|
||||
|
||||
|
||||
/*!
|
||||
* Return the name of the executable
|
||||
* @return Returns the name of the executable (usually the full path)
|
||||
*/
|
||||
std::string getExecutable();
|
||||
|
||||
|
||||
/*!
|
||||
* Return the search path for the symbols
|
||||
* @return Returns the search path for the symbols
|
||||
*/
|
||||
std::string getSymPaths();
|
||||
|
||||
|
||||
//!< Terminate type
|
||||
enum class terminateType { signal, exception };
|
||||
|
||||
/*!
|
||||
* Set the error handlers
|
||||
* @param[in] Function to terminate the program: abort(msg,type)
|
||||
*/
|
||||
void setErrorHandlers( std::function<void( std::string, terminateType )> abort );
|
||||
|
||||
|
||||
/*!
|
||||
* Set the given signals to the handler
|
||||
* @param[in] Function to terminate the program: abort(msg,type)
|
||||
*/
|
||||
void setSignals( const std::vector<int>& signals, void (*handler) (int) );
|
||||
|
||||
|
||||
//! Clear a signal set by setSignals
|
||||
void clearSignal( int signal );
|
||||
|
||||
|
||||
//! Clear all signals set by setSignals
|
||||
void clearSignals( );
|
||||
|
||||
|
||||
//! Return a list of all signals that can be caught
|
||||
std::vector<int> allSignalsToCatch( );
|
||||
|
||||
//! Return a default list of signals to catch
|
||||
std::vector<int> defaultSignalsToCatch( );
|
||||
|
||||
|
||||
//! Get a list of the active threads
|
||||
std::set<std::thread::native_handle_type> activeThreads( );
|
||||
|
||||
//! Get a handle to this thread
|
||||
std::thread::native_handle_type thisThread( );
|
||||
|
||||
|
||||
//! Initialize globalCallStack functionallity
|
||||
void globalCallStackInitialize( MPI_Comm comm );
|
||||
|
||||
//! Clean up globalCallStack functionallity
|
||||
void globalCallStackFinalize( );
|
||||
|
||||
|
||||
/*!
|
||||
* @brief Call system command
|
||||
* @details This function calls a system command, waits for the program
|
||||
* to execute, captures and returns the output and exit code.
|
||||
* @param[in] cmd Command to execute
|
||||
* @param[out] exit_code Exit code returned from child process
|
||||
* @return Returns string containing the output
|
||||
*/
|
||||
std::string exec( const std::string& cmd, int& exit_code );
|
||||
int getSymbols( std::vector<void*>& address, std::vector<char>& type, std::vector<std::string>& obj );
|
||||
|
||||
|
||||
} // namespace StackTrace
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@ -236,9 +236,9 @@ size_t Utilities::getMemoryUsage()
|
||||
size_t N_bytes = 0;
|
||||
#if defined(USE_LINUX)
|
||||
struct mallinfo meminfo = mallinfo();
|
||||
size_t size_hblkhd = static_cast<unsigned int>( meminfo.hblkhd );
|
||||
size_t size_uordblks = static_cast<unsigned int>( meminfo.uordblks );
|
||||
N_bytes = size_hblkhd + size_uordblks;
|
||||
size_t size_hblkhd = static_cast<size_t>( meminfo.hblkhd );
|
||||
size_t size_uordblks = static_cast<size_t>( meminfo.uordblks );
|
||||
N_bytes = static_cast<size_t>( size_hblkhd + size_uordblks );
|
||||
#elif defined(USE_MAC)
|
||||
struct task_basic_info t_info;
|
||||
mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
|
||||
@ -347,11 +347,3 @@ std::vector<int> Utilities::factor(size_t number)
|
||||
std::sort( factors.begin(), factors.end() );
|
||||
return factors;
|
||||
}
|
||||
|
||||
|
||||
// Dummy function to prevent compiler from optimizing away variable
|
||||
void Utilities::nullUse( void* data )
|
||||
{
|
||||
NULL_USE(data);
|
||||
}
|
||||
|
||||
|
||||
@ -61,9 +61,6 @@ namespace Utilities
|
||||
//! Factor a number into it's prime factors
|
||||
std::vector<int> factor(size_t number);
|
||||
|
||||
//! Print AMP Banner
|
||||
void nullUse( void* );
|
||||
|
||||
} // namespace Utilities
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user