OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updating load balance for analysis threads

Browse Source
This commit is contained in:
Mark Berrill
2018-03-09 12:59:59 -05:00
parent 5772b4cdb0
commit cd2c7c804c
3 changed files with 48 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
analysis/runAnalysis.cpp
View File

@@ -267,7 +267,7 @@ runAnalysis::commWrapper runAnalysis::getComm( )
/******************************************************************
* Constructor/Destructors *
******************************************************************/
runAnalysis::runAnalysis( int N_threads, int restart_interval, int analysis_interval,
runAnalysis::runAnalysis( int restart_interval, int analysis_interval,
int blobid_interval, const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& Dm,
int Np, int Nx, int Ny, int Nz, double Lx, double Ly, double Lz, bool pBC, double beta, double err,
IntArray Map, const std::string& LocalRestartFile ):
@@ -276,7 +276,6 @@ runAnalysis::runAnalysis( int N_threads, int restart_interval, int analysis_inte
d_analysis_interval( analysis_interval ),
d_blobid_interval( blobid_interval ),
d_beta( beta ),
d_tpool( N_threads ),
d_ScaLBL_Comm( ScaLBL_Comm ),
d_rank_info( rank_info ),
d_Map( Map ),
@@ -362,12 +361,33 @@ void print( const std::vector<int>& ids )
printf(", %i",ids[i]);
printf("\n");
}
void runAnalysis::setAffinities( const std::string& method )
void runAnalysis::createThreads( const std::string& method, int N_threads )
{
int N_procs = d_tpool.getNumberOfProcessors();
int proc = d_tpool.getCurrentProcessor();
auto affinity = d_tpool.getProcessAffinity();
// Check if we are not using analysis threads
if ( method == "none" )
return;
// Check if we have thread support
int thread_support;
MPI_Query_thread( &thread_support );
if ( thread_support < MPI_THREAD_MULTIPLE ) {
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
return;
}
// Create the threads
const auto cores = d_tpool.getProcessAffinity();
if ( cores.empty() ) {
// We were not able to get the cores for the process
d_tpool.setNumThreads( N_threads );
} else if ( method == "default" ) {
// Create the given number of threads, but let the OS manage affinities
d_tpool.setNumThreads( N_threads );
} else if ( method == "independent" ) {
int N = cores.size() - 1;
d_tpool.setNumThreads( N );
d_tpool.setThreadAffinity( { cores[0] } );
for ( int i=0; i<N; i++)
d_tpool.setThreadAffinity( i, { cores[i+1] } );
}
// Print the current affinities
if ( d_rank == 0 ) {
printf("Affinities - rank 0:\n");

15
analysis/runAnalysis.h
View File

@@ -23,7 +23,7 @@ class runAnalysis
public:
//! Constructor
runAnalysis( int N_threads, int restart_interval, int analysis_interval, int blobid_interval,
runAnalysis( int restart_interval, int analysis_interval, int blobid_interval,
const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& dm,
int Np, int Nx, int Ny, int Nz, double Lx, double Ly, double Lz, bool pBC, double beta, double err,
IntArray Map, const std::string& LocalRestartFile );
@@ -40,11 +40,18 @@ public:
/*!
* \brief Set the affinities
* \details This function will set the affinity of this thread and all analysis threads
* \details This function will create the analysis threads and set the affinity
* of this thread and all analysis threads. If MPI_THREAD_MULTIPLE is not
* enabled, the analysis threads will be disabled and the analysis will run in the current thread.
* @param[in] method Method used to control the affinities:
* none - Don't do anything
* none - Don't use threads (runs all analysis in the current thread)
* default - Create the specified number of threads, but don't load balance
* independent - Create the necessary number of threads to fill all cpus,
* and set the affinities based on the current process such
* that all threads run on independent cores
* @param[in] N_threads Number of threads, only used by some of the methods
*/
void setAffinities( const std::string& method = "none" );
void createThreads( const std::string& method = "default", int N_threads = 4 );
private:

27
tests/lbpm_color_macro_simulator.cpp
View File

@@ -34,8 +34,6 @@ int main(int argc, char **argv)
MPI_Comm_dup(MPI_COMM_WORLD,&comm);
int rank = comm_rank(comm);
int nprocs = comm_size(comm);
if ( rank==0 && provided_thread_support<MPI_THREAD_MULTIPLE )
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
// parallel domain size (# of sub-domains)
@@ -58,16 +56,15 @@ int main(int argc, char **argv)
PROFILE_START("Main");
Utilities::setErrorHandlers();
int ANALYSIS_INTERVAL;
int BLOBID_INTERVAL;
if (argc == 3){
ANALYSIS_INTERVAL=atoi(argv[1]);
BLOBID_INTERVAL = atoi(argv[2]);
}
else{
ANALYSIS_INTERVAL=1000;
BLOBID_INTERVAL = 1000;
int ANALYSIS_INTERVAL = 1000;
int BLOBID_INTERVAL = 1000;
std::string analysis_method = "independent";
if (argc >= 3) {
ANALYSIS_INTERVAL = atoi(argv[1]);
BLOBID_INTERVAL = atoi(argv[2]);
}
if (argc >= 4)
analysis_method = std::string(argv[3]);
// Variables that specify the computational domain
string FILENAME;
@@ -524,16 +521,12 @@ int main(int argc, char **argv)
err = 1.0;
double sat_w_previous = 1.01; // slightly impossible value!
if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol);
// Create the thread pool
int N_threads = 4;
if ( provided_thread_support < MPI_THREAD_MULTIPLE )
N_threads = 0;
//************ MAIN ITERATION LOOP ***************************************/
PROFILE_START("Loop");
runAnalysis analysis( N_threads, RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL,
runAnalysis analysis( RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL,
rank_info, ScaLBL_Comm, Dm, Np, Nx, Ny, Nz, Lx, Ly, Lz, pBC, beta, err, Map, LocalRestartFile );
analysis.setAffinities( "none" );
analysis.createThreads( analysis_method, 4 );
while (timestep < timestepMax && err > tol ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");