OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-01-27 04:36:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

Fix initialization and stopping Damaris.

Browse Source 
With this, flow no longer hangs on exit.
This commit is contained in:
Atgeirr Flø Rasmussen 2022-10-06 16:35:35 +02:00 committed by Elyes Ahmed
parent e347959d79
commit fae71df32d
1 changed files with 49 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
opm/simulators/flow/Main.hpp
View File

@ -169,13 +169,22 @@ public:
#endif // HAVE_MPI
EclGenericVanguard::setCommunication(nullptr);
#if HAVE_DAMARIS
int err = damaris_finalize();
if (err != DAMARIS_OK ) {
std::cerr << "ERROR: Damaris library produced an error result for damaris_initialize()" << std::endl;
#if HAVE_DAMARIS
if (enableDamarisOutput_) {
int err;
if (isSimulationRank_) {
err = damaris_stop();
if (err != DAMARIS_OK) {
std::cerr << "ERROR: Damaris library produced an error result for damaris_stop()" << std::endl;
}
}
err = damaris_finalize();
if (err != DAMARIS_OK) {
std::cerr << "ERROR: Damaris library produced an error result for damaris_finalize()" << std::endl;
}
}
#endif
#endif // HAVE_DAMARIS
#if HAVE_MPI && !HAVE_DUNE_FEM
MPI_Finalize();
@ -388,11 +397,6 @@ private:
externalSetupTimer.start();
handleVersionCmdLine_(argc_, argv_);
#if HAVE_DUNE_FEM
int mpiRank = Dune::Fem::MPIManager::rank();
#else
int mpiRank = EclGenericVanguard::comm().rank();
#endif // HAVE_DUNE_FEM
// we always want to use the default locale, and thus spare us the trouble
// with incorrect locale settings.
@ -421,11 +425,6 @@ private:
return false; // Whether to run the simulator
}
FileOutputMode outputMode = FileOutputMode::OUTPUT_NONE;
outputCout_ = false;
if (mpiRank == 0)
outputCout_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableTerminalOutput);
std::string deckFilename;
std::string outputDir;
if ( eclipseState_ ) {
@ -434,8 +433,40 @@ private:
}
else {
deckFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, EclDeckFileName);
outputDir = EWOMS_GET_PARAM(PreTypeTag, std::string, OutputDir);
}
#if HAVE_DAMARIS
enableDamarisOutput_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutput);
if (enableDamarisOutput_) {
// By default EnableAsyncDamarisOutput is false so FilePerCore Mode is used in Damaris so all the
// simulation results in each node are aggregated by dedicated cores and stored asynchronously at the end of
// each iteration.
const bool enableAsyncDamarisOutput = EWOMS_GET_PARAM(PreTypeTag, bool, EnableAsyncDamarisOutput);
// Using the ModifyModel class to set the XML file for Damaris.
// If EnableAsyncDamarisOutput is enabled, all simulation results will
// be written into one single file for each iteration using Parallel HDF5.
DamarisOutput::initializeDamaris(EclGenericVanguard::comm(), EclGenericVanguard::comm().rank(), outputDir, enableAsyncDamarisOutput);
int is_client;
MPI_Comm new_comm;
int err = damaris_start(&is_client);
isSimulationRank_ = (is_client > 0);
if (isSimulationRank_) {
damaris_client_comm_get(&new_comm);
EclGenericVanguard::setCommunication(std::make_unique<Parallel::Communication>(new_comm));
} else {
return false;
}
}
#endif // HAVE_DAMARIS
int mpiRank = EclGenericVanguard::comm().rank();
FileOutputMode outputMode = FileOutputMode::OUTPUT_NONE;
outputCout_ = false;
if (mpiRank == 0)
outputCout_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableTerminalOutput);
if (deckFilename.empty()) {
if (mpiRank == 0) {
std::cerr << "No input case given. Try '--help' for a usage description.\n";
@ -465,9 +496,6 @@ private:
try {
auto python = std::make_shared<Python>();
const bool init_from_restart_file = !EWOMS_GET_PARAM(PreTypeTag, bool, SchedRestart);
if (outputDir.empty())
outputDir = EWOMS_GET_PARAM(PreTypeTag, std::string, OutputDir);
const bool allRanksDbgPrtLog = EWOMS_GET_PARAM(PreTypeTag, bool,
EnableLoggingFalloutWarning);
outputMode = setupLogging(mpiRank,
@ -495,27 +523,6 @@ private:
if (output_param >= 0)
outputInterval = output_param;
#if HAVE_DAMARIS
if (EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutput)) {
// By default EnableAsyncDamarisOutput is false so FilePerCore Mode is used in Damaris so all the simulation results
// in each node are aggregated by dedicated cores and stored asynchronously at the end of each iteration.
if (EWOMS_GET_PARAM(PreTypeTag, bool, EnableAsyncDamarisOutput))
enableAsyncDamarisOutput = true;
// Using the ModifyModel class to set the XML file for Damaris.
// If EnableAsyncDamarisOutput is enabled, all simulation results will
// be written into one single file for each iteration using Parallel HDF5.
DamarisOutput::initializeDamaris(EclGenericVanguard::comm(), mpiRank, outputDir, enableAsyncDamarisOutput);
int is_client;
MPI_Comm new_comm;
int err = damaris_start(&is_client) ;
isSimulationRank_ = (is_client > 0) ;
if (isSimulationRank_) {
damaris_client_comm_get (&new_comm) ;
EclGenericVanguard::setCommunication(std::make_unique<Parallel::Communication>(new_comm));
}
}
#endif // HAVE_DAMARIS
readDeck(EclGenericVanguard::comm(), deckFilename, deck_, eclipseState_, schedule_, udqState_, actionState_, wtestState_,
summaryConfig_, nullptr, python, std::move(parseContext),
init_from_restart_file, outputCout_, outputInterval);
@ -819,8 +826,8 @@ private:
// To demonstrate run with non_world_comm
bool test_split_comm_ = false;
bool isSimulationRank_ = true;
// To use Damaris with parallel HDF5: Asynchronous Output using a separate core
bool enableAsyncDamarisOutput = false;
bool enableDamarisOutput_ = false;
};
} // namespace Opm