From e50a099c13eab93a6a302c82c0824c7f4fb89bcc Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 12:20:21 -0400 Subject: [PATCH 1/5] cleaning up barriers in color model --- models/ColorModel.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index f6d15b43..a62ec927 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -716,7 +716,8 @@ void ScaLBL_ColorModel::Run(){ } ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); // *************EVEN TIMESTEP************* timestep++; @@ -751,10 +752,9 @@ void ScaLBL_ColorModel::Run(){ } ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); //************************************************************************ - - MPI_Barrier(comm); PROFILE_STOP("Update"); if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition > 0){ @@ -763,7 +763,6 @@ void ScaLBL_ColorModel::Run(){ // Run the analysis analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); - // allow initial ramp-up to get closer to steady state if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){ analysis.finish(); @@ -1002,17 +1001,17 @@ void ScaLBL_ColorModel::Run(){ //morph_delta *= (-1.0); REVERSE_FLOW_DIRECTION = false; } - MPI_Barrier(comm); } morph_timesteps += analysis_interval; } + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); } analysis.finish(); PROFILE_STOP("Loop"); PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep @@ -1062,12 +1061,12 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){ if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); - MPI_Barrier(comm); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); ScaLBL_D3Q19_Init(fq, Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - MPI_Barrier(comm); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); @@ -1442,7 +1441,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta if (USE_CONNECTED_NWP){ BlobIDstruct new_index; ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - MPI_Barrier(comm); + MPI_Barrier(Dm->comm); // only operate on component "0" count = 0.0; From 7ef292e2fcc2a1864a626c1ef5475b6c708eda4e Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 12:23:00 -0400 Subject: [PATCH 2/5] cleaning up barriers in color model --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index a62ec927..b86c0918 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1441,7 +1441,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta if (USE_CONNECTED_NWP){ BlobIDstruct new_index; ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - MPI_Barrier(Dm->comm); + MPI_Barrier(Dm->Comm); // only operate on component "0" count = 0.0; From 64a19a718bc5a6725ef11bb249c5352745549646 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 12:26:55 -0400 Subject: [PATCH 3/5] make ScaLBL communicator public for --- common/ScaLBL.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index a50ab7ed..0d2ee0cf 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -134,6 +134,7 @@ public: //ScaLBL_Communicator(Domain &Dm, IntArray &Map); ~ScaLBL_Communicator(); //...................................................................................... + MPI_Comm MPI_COMM_SCALBL; // MPI Communicator unsigned long int CommunicationCount,SendCount,RecvCount; int Nx,Ny,Nz,N; int BoundaryCondition; @@ -207,7 +208,6 @@ private: // Give the object it's own MPI communicator RankInfoStruct rank_info; MPI_Group Group; // Group of processors associated with this domain - MPI_Comm MPI_COMM_SCALBL; // MPI Communicator for this domain MPI_Request req1[18],req2[18]; MPI_Status stat1[18],stat2[18]; //...................................................................................... From 50b84071456fb6806c734735730c6982384420e9 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 13:13:57 -0400 Subject: [PATCH 4/5] clean up ScaLBL barriers --- cpu/Extras.cpp | 2 +- gpu/Extras.cu | 1 + models/ColorModel.cpp | 4 +--- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cpu/Extras.cpp b/cpu/Extras.cpp index 71f5c04a..efe820d3 100644 --- a/cpu/Extras.cpp +++ b/cpu/Extras.cpp @@ -49,5 +49,5 @@ extern "C" void ScaLBL_CopyToZeroCopy(void* dest, const void* source, size_t siz } extern "C" void ScaLBL_DeviceBarrier(){ -// cudaDeviceSynchronize(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL) } diff --git a/gpu/Extras.cu b/gpu/Extras.cu index 8aeedc87..cd9c265c 100644 --- a/gpu/Extras.cu +++ b/gpu/Extras.cu @@ -59,4 +59,5 @@ extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size){ extern "C" void ScaLBL_DeviceBarrier(){ cudaDeviceSynchronize(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL) } diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index b86c0918..57e50411 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -717,7 +717,7 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + ; // *************EVEN TIMESTEP************* timestep++; @@ -753,7 +753,6 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); //************************************************************************ PROFILE_STOP("Update"); @@ -1011,7 +1010,6 @@ void ScaLBL_ColorModel::Run(){ PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep From f12f8154b12b5aa4fc2a124a1a615ec3735bb176 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 13:15:24 -0400 Subject: [PATCH 5/5] Revert "clean up ScaLBL barriers" This reverts commit 50b84071456fb6806c734735730c6982384420e9. --- cpu/Extras.cpp | 2 +- gpu/Extras.cu | 1 - models/ColorModel.cpp | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cpu/Extras.cpp b/cpu/Extras.cpp index efe820d3..71f5c04a 100644 --- a/cpu/Extras.cpp +++ b/cpu/Extras.cpp @@ -49,5 +49,5 @@ extern "C" void ScaLBL_CopyToZeroCopy(void* dest, const void* source, size_t siz } extern "C" void ScaLBL_DeviceBarrier(){ - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL) +// cudaDeviceSynchronize(); } diff --git a/gpu/Extras.cu b/gpu/Extras.cu index cd9c265c..8aeedc87 100644 --- a/gpu/Extras.cu +++ b/gpu/Extras.cu @@ -59,5 +59,4 @@ extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size){ extern "C" void ScaLBL_DeviceBarrier(){ cudaDeviceSynchronize(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL) } diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 57e50411..b86c0918 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -717,7 +717,7 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); - ; + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); // *************EVEN TIMESTEP************* timestep++; @@ -753,6 +753,7 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); //************************************************************************ PROFILE_STOP("Update"); @@ -1010,6 +1011,7 @@ void ScaLBL_ColorModel::Run(){ PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep