disable MPI thread multiple for crusher

This commit is contained in:
James E McClure 2022-02-09 16:40:30 -05:00
parent 05712e6a16
commit d1a45a3b1e
3 changed files with 19 additions and 7 deletions

View File

@ -270,17 +270,19 @@ int main(int argc, char **argv)
//....................................................................... //.......................................................................
//........................................................................... //...........................................................................
comm.barrier(); //comm.barrier();
if (rank == 0) cout << "Domain set." << endl; if (rank == 0) cout << "Domain set." << endl;
//........................................................................... //...........................................................................
cout << flush;
//........................................................................... //...........................................................................
if (rank==0) printf ("Create ScaLBL_Communicator \n"); if (rank==0) printf ("Create ScaLBL_Communicator \n");
cout << flush;
// Create a communicator for the device (will use optimized layout) // Create a communicator for the device (will use optimized layout)
ScaLBL_Communicator ScaLBL_Comm(Dm); ScaLBL_Communicator ScaLBL_Comm(Dm);
int Npad=(Np/16 + 2)*16; int Npad=(Np/16 + 2)*16;
if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N);
cout << flush;
auto neighborList= new int[18*Npad]; auto neighborList= new int[18*Npad];
IntArray Map(Nx,Ny,Nz); IntArray Map(Nx,Ny,Nz);
Map.fill(-2); Map.fill(-2);
@ -291,7 +293,8 @@ int main(int argc, char **argv)
//......................device distributions................................. //......................device distributions.................................
dist_mem_size = Np*sizeof(double); dist_mem_size = Np*sizeof(double);
if (rank==0) printf ("Allocating distributions \n"); if (rank==0) printf ("Allocating distributions \n");
cout << flush;
int *NeighborList; int *NeighborList;
int *dvcMap; int *dvcMap;
double *fq; double *fq;
@ -321,6 +324,9 @@ int main(int argc, char **argv)
ScaLBL_DeviceBarrier(); ScaLBL_DeviceBarrier();
delete [] TmpMap; delete [] TmpMap;
if (rank==0) printf("Map is copied to GPU \n");
cout << flush;
//........................................................................... //...........................................................................
/* // Write the communcation structure into a file for debugging /* // Write the communcation structure into a file for debugging
@ -352,11 +358,13 @@ int main(int argc, char **argv)
fclose(CommFile); fclose(CommFile);
*/ */
if (rank==0) printf("Setting the distributions, size = : %i\n", Np); if (rank==0) printf("Setting the distributions, size = : %i\n", Np);
cout << flush;
//........................................................................... //...........................................................................
GlobalFlipScaLBL_D3Q19_Init(fq_host, Map, Np, Nx-2, Ny-2, Nz-2, iproc,jproc,kproc,nprocx,nprocy,nprocz); GlobalFlipScaLBL_D3Q19_Init(fq_host, Map, Np, Nx-2, Ny-2, Nz-2, iproc,jproc,kproc,nprocx,nprocy,nprocz);
ScaLBL_CopyToDevice(fq, fq_host, 19*dist_mem_size); ScaLBL_CopyToDevice(fq, fq_host, 19*dist_mem_size);
ScaLBL_DeviceBarrier(); ScaLBL_DeviceBarrier();
comm.barrier(); //comm.barrier();
//************************************************************************* //*************************************************************************
// First timestep // First timestep
ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
@ -376,6 +384,7 @@ int main(int argc, char **argv)
int timestep = 0; int timestep = 0;
if (rank==0) printf("********************************************************\n"); if (rank==0) printf("********************************************************\n");
if (rank==0) printf("No. of timesteps for timing: %i \n", 100); if (rank==0) printf("No. of timesteps for timing: %i \n", 100);
cout << flush;
//.......create and start timer............ //.......create and start timer............
double starttime,stoptime,cputime; double starttime,stoptime,cputime;
@ -426,6 +435,8 @@ int main(int argc, char **argv)
// communication bandwidth includes both send and recieve // communication bandwidth includes both send and recieve
if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9); if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9);
if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9); if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9);
cout << flush;
} }
// **************************************************** // ****************************************************
cout << fflush; cout << fflush;

View File

@ -23,7 +23,7 @@ int main( int argc, char **argv )
{ {
// Initialize // Initialize
Utilities::startup( argc, argv ); Utilities::startup( argc, argv, false );
{ // Limit scope so variables that contain communicators will free before MPI_Finialize { // Limit scope so variables that contain communicators will free before MPI_Finialize
@ -198,7 +198,7 @@ int main( int argc, char **argv )
} // Limit scope so variables that contain communicators will free before MPI_Finialize } // Limit scope so variables that contain communicators will free before MPI_Finialize
cout << flush;
Utilities::shutdown(); Utilities::shutdown();
return 0; return 0;
} }

View File

@ -24,7 +24,7 @@ using namespace std;
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
// Initialize MPI // Initialize MPI
Utilities::startup( argc, argv ); Utilities::startup( argc, argv, false );
Utilities::MPI comm( MPI_COMM_WORLD ); Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank(); int rank = comm.getRank();
int nprocs = comm.getSize(); int nprocs = comm.getSize();
@ -49,6 +49,7 @@ int main(int argc, char **argv)
MRT.Initialize(); // initializing the model will set initial conditions for variables MRT.Initialize(); // initializing the model will set initial conditions for variables
MRT.Run(); MRT.Run();
MRT.VelocityField(); MRT.VelocityField();
cout << flush;
} }
Utilities::shutdown(); Utilities::shutdown();
} }