OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2024-11-21 16:57:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

Replacing use of MPI_COMM_WORLD with a variable communicator.

Browse Source
This commit is contained in:
Elyes Ahmed 2021-05-25 12:57:11 +02:00 committed by Atgeirr Flø Rasmussen
parent 61ef539bf5
commit f53c597f90
48 changed files with 584 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ebos/collecttoiorank.cc
View File

@ -757,7 +757,7 @@ CollectDataToIORank(const Grid& grid, const EquilGrid* equilGrid,
const GridView& localGridView,
const Dune::CartesianIndexMapper<Grid>& cartMapper,
const Dune::CartesianIndexMapper<EquilGrid>* equilCartMapper)
: toIORankComm_()
: toIORankComm_(grid.comm())
{
// index maps only have to be build when reordering is needed
if (!needsReordering && !isParallel())

18
ebos/eclgenericcpgridvanguard.cc
View File

@ -58,7 +58,7 @@ template<class ElementMapper, class GridView, class Scalar>
EclGenericCpGridVanguard<ElementMapper,GridView,Scalar>::EclGenericCpGridVanguard()
{
#if HAVE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
MPI_Comm_rank(EclGenericVanguard::comm(), &mpiRank);
#else
mpiRank = 0;
#endif
@ -85,7 +85,7 @@ void EclGenericCpGridVanguard<ElementMapper,GridView,Scalar>::doLoadBalance_(Dun
EclGenericVanguard::ParallelWellStruct& parallelWells)
{
int mpiSize = 1;
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_size(grid_->comm(), &mpiSize);
if (mpiSize > 1) {
// the CpGrid's loadBalance() method likes to have the transmissibilities as
@ -188,7 +188,7 @@ template<class ElementMapper, class GridView, class Scalar>
void EclGenericCpGridVanguard<ElementMapper,GridView,Scalar>::distributeFieldProps_(EclipseState& eclState1)
{
int mpiSize = 1;
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_size(grid_->comm(), &mpiSize);
if (mpiSize > 1) {
try
@ -230,7 +230,12 @@ void EclGenericCpGridVanguard<ElementMapper,GridView,Scalar>::doCreateGrids_(Ecl
OpmLog::info("\nProcessing grid");
}
#if HAVE_MPI
grid_.reset(new Dune::CpGrid(EclGenericVanguard::comm()));
#else
grid_.reset(new Dune::CpGrid());
#endif
const auto& removed_cells = grid_->processEclipseFormat(input_grid,
&eclState,
/*isPeriodic=*/false,
@ -262,12 +267,13 @@ void EclGenericCpGridVanguard<ElementMapper,GridView,Scalar>::doCreateGrids_(Ecl
{
const bool has_numerical_aquifer = eclState.aquifer().hasNumericalAquifer();
int mpiSize = 1;
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_size(grid_->comm(), &mpiSize);
// when there is numerical aquifers, new NNC are generated during grid processing
// we need to pass the NNC from root process to other processes
if (has_numerical_aquifer && mpiSize > 1) {
auto nnc_input = eclState.getInputNNC();
EclMpiSerializer ser(Dune::MPIHelper::getCollectiveCommunication());
const auto& comm_nnc = Opm::Parallel::Communication();
EclMpiSerializer ser(comm_nnc);
ser.broadcast(nnc_input);
if (mpiRank > 0) {
eclState.setInputNNC(nnc_input);
@ -312,7 +318,7 @@ void EclGenericCpGridVanguard<ElementMapper,GridView,Scalar>::doFilterConnection
{
// Broadcast another time to remove inactive peforations on
// slave processors.
eclScheduleBroadcast(schedule);
eclScheduleBroadcast(EclGenericVanguard::comm(), schedule);
}
catch(const std::exception& broadcast_error)
{

11
ebos/eclgenericvanguard.cc
View File

@ -62,7 +62,7 @@ std::shared_ptr<Schedule> EclGenericVanguard::externalEclSchedule_;
std::shared_ptr<SummaryConfig> EclGenericVanguard::externalEclSummaryConfig_;
std::unique_ptr<UDQState> EclGenericVanguard::externalUDQState_;
std::unique_ptr<Action::State> EclGenericVanguard::externalActionState_;
std::unique_ptr<EclGenericVanguard::CommunicationType> EclGenericVanguard::comm_;
std::unique_ptr<Parallel::Communication> EclGenericVanguard::comm_;
EclGenericVanguard::EclGenericVanguard()
: python(std::make_shared<Python>())
@ -295,7 +295,7 @@ void EclGenericVanguard::init()
parseContext_ = createParseContext(ignoredKeywords_, eclStrictParsing_);
}
readDeck(myRank, fileName_, deck_, eclState_, eclSchedule_, udqState_, actionState_,
readDeck(EclGenericVanguard::comm(), fileName_, deck_, eclState_, eclSchedule_, udqState_, actionState_,
eclSummaryConfig_, std::move(errorGuard), python,
std::move(parseContext_), /* initFromRestart = */ false,
/* checkDeck = */ enableExperiments_, outputInterval_);
@ -338,11 +338,8 @@ void EclGenericVanguard::init()
}
}
}
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
const auto& comm = Dune::MPIHelper::getCommunication();
#else
const auto& comm = Dune::MPIHelper::getCollectiveCommunication();
#endif
const auto& comm = Parallel::Communication();
hasMsWell = comm.max(hasMsWell);
if (hasMsWell)

23
ebos/eclgenericvanguard.hh
View File

@ -44,6 +44,14 @@
#include <utility>
#include <vector>
namespace Opm::Parallel {
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
}
namespace Opm {
namespace Action { class State; }
@ -60,13 +68,8 @@ class UDQState;
class EclGenericVanguard {
public:
using ParallelWellStruct = std::vector<std::pair<std::string,bool>>;
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using CommunicationType = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using CommunicationType = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
using ParallelWellStruct = std::vector<std::pair<std::string,bool>>;
/*!
* \brief Constructor.
@ -277,11 +280,11 @@ public:
{ return parallelWells_; }
//! \brief Set global communication.
static void setCommunication(std::unique_ptr<CommunicationType> comm)
static void setCommunication(std::unique_ptr<Opm::Parallel::Communication> comm)
{ comm_ = std::move(comm); }
//! \brief Obtain global communicator.
static CommunicationType& comm()
static Parallel::Communication& comm()
{
assert(comm_);
return *comm_;
@ -310,7 +313,7 @@ protected:
static bool externalDeckSet_;
static std::unique_ptr<UDQState> externalUDQState_;
static std::unique_ptr<Action::State> externalActionState_;
static std::unique_ptr<CommunicationType> comm_;
static std::unique_ptr<Parallel::Communication> comm_;
std::string caseName_;
std::string fileName_;

4
ebos/eclgenericwriter.cc
View File

@ -512,11 +512,11 @@ evalSummary(int reportStepNum,
if (collectToIORank_.isParallel()) {
#ifdef HAVE_MPI
unsigned long buffer_size = buffer.size();
MPI_Bcast(&buffer_size, 1, MPI_UNSIGNED_LONG, collectToIORank_.ioRank, MPI_COMM_WORLD);
MPI_Bcast(&buffer_size, 1, MPI_UNSIGNED_LONG, collectToIORank_.ioRank, grid_.comm());
if (!collectToIORank_.isIORank())
buffer.resize( buffer_size );
MPI_Bcast(buffer.data(), buffer_size, MPI_CHAR, collectToIORank_.ioRank, MPI_COMM_WORLD);
MPI_Bcast(buffer.data(), buffer_size, MPI_CHAR, collectToIORank_.ioRank, grid_.comm());
if (!collectToIORank_.isIORank()) {
SummaryState& st = summaryState;
st.deserialize(buffer);

3
ebos/eclmpiserializer.hh
View File

@ -21,6 +21,7 @@
#ifndef ECL_MPI_SERIALIZER_HH
#define ECL_MPI_SERIALIZER_HH
#include <dune/common/version.hh>
#include <opm/simulators/utils/ParallelRestart.hpp>
#include <optional>
@ -39,7 +40,7 @@ class EclMpiSerializer {
public:
//! \brief Constructor.
//! \param comm The global communicator to broadcast using
explicit EclMpiSerializer(Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator> comm) :
explicit EclMpiSerializer(Opm::Parallel::Communication comm) :
m_comm(comm)
{}

13
ebos/eclproblem.hh
View File

@ -2121,7 +2121,7 @@ private:
Scalar>(fs, iq.pvtRegionIndex());
}
}
OPM_END_PARALLEL_TRY_CATCH("EclProblem::_updateCompositionLayers() failed: ");
OPM_END_PARALLEL_TRY_CATCH("EclProblem::_updateCompositionLayers() failed: ", this->simulator().vanguard().grid().comm());
}
bool updateMaxOilSaturation_()
@ -2150,7 +2150,7 @@ private:
this->maxOilSaturation_[compressedDofIdx] = std::max(this->maxOilSaturation_[compressedDofIdx], So);
}
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateMayOilSaturation() failed:");
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateMayOilSaturation() failed:", vanguard.grid().comm());
// we need to invalidate the intensive quantities cache here because the
// derivatives of Rs and Rv will most likely have changed
return true;
@ -2184,7 +2184,7 @@ private:
Scalar Sw = decay<Scalar>(fs.saturation(waterPhaseIdx));
this->maxWaterSaturation_[compressedDofIdx] = std::max(this->maxWaterSaturation_[compressedDofIdx], Sw);
}
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateMayWaterSaturation() failed: ");
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateMayWaterSaturation() failed: ", vanguard.grid().comm());
return true;
}
@ -2214,8 +2214,7 @@ private:
std::min(this->minOilPressure_[compressedDofIdx],
getValue(fs.pressure(oilPhaseIdx)));
}
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateMinPressure_() failed: ");
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateMinPressure_() failed: ", this->simulator().vanguard().grid().comm());
return true;
}
@ -2658,7 +2657,7 @@ private:
const auto& intQuants = elemCtx.intensiveQuantities(/*spaceIdx=*/0, /*timeIdx=*/0);
materialLawManager_->updateHysteresis(intQuants.fluidState(), compressedDofIdx);
}
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateHyteresis_(): ");
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateHyteresis_(): ", vanguard.grid().comm());
return true;
}
@ -2683,7 +2682,7 @@ private:
this->maxPolymerAdsorption_[compressedDofIdx] = std::max(this->maxPolymerAdsorption_[compressedDofIdx],
scalarValue(intQuants.polymerAdsorption()));
}
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateMaxPolymerAdsorption_(): ");
OPM_END_PARALLEL_TRY_CATCH("EclProblem::updateMaxPolymerAdsorption_(): ", vanguard.grid().comm());
}
struct PffDofData_

2
ebos/eclwellmanager.hh
View File

@ -420,7 +420,7 @@ public:
for (size_t wellIdx = 0; wellIdx < wellSize; ++wellIdx)
wells_[wellIdx]->beginIterationAccumulate(elemCtx, /*timeIdx=*/0);
}
OPM_END_PARALLEL_TRY_CATCH("EclWellManager::beginIteration() failed: ");
OPM_END_PARALLEL_TRY_CATCH("EclWellManager::beginIteration() failed: ", simulator_.vanguard().grid().comm());
// call the postprocessing routines
for (size_t wellIdx = 0; wellIdx < wellSize; ++wellIdx)

2
ebos/eclwriter.hh
View File

@ -393,7 +393,7 @@ private:
eclOutputModule_->processElement(elemCtx);
}
OPM_END_PARALLEL_TRY_CATCH("EclWriter::prepareLocalCellData() failed: ")
OPM_END_PARALLEL_TRY_CATCH("EclWriter::prepareLocalCellData() failed: ", simulator_.vanguard().grid().comm())
}
Simulator& simulator_;

3
opm/simulators/aquifers/AquiferInterface.hpp
View File

@ -128,7 +128,8 @@ public:
const auto& iq = elemCtx.intensiveQuantities(0, 0);
pressure_previous_[idx] = getValue(iq.fluidState().pressure(phaseIdx_()));
}
OPM_END_PARALLEL_TRY_CATCH("AquiferInterface::beginTimeStep() failed: ");
OPM_END_PARALLEL_TRY_CATCH("AquiferInterface::beginTimeStep() failed: ", ebos_simulator_.vanguard().grid().comm());
}
template <class Context>

3
opm/simulators/aquifers/AquiferNumerical.hpp
View File

@ -228,8 +228,7 @@ private:
cell_pressure[idx] = water_pressure_reservoir;
}
OPM_END_PARALLEL_TRY_CATCH("AquiferNumerical::calculateAquiferPressure() failed: ");
OPM_END_PARALLEL_TRY_CATCH("AquiferNumerical::calculateAquiferPressure() failed: ", this->ebos_simulator_.vanguard().grid().comm());
const auto& comm = this->ebos_simulator_.vanguard().grid().comm();
comm.sum(&sum_pressure_watervolume, 1);
comm.sum(&sum_watervolume, 1);

4
opm/simulators/flow/BlackoilModelEbos.hpp
View File

@ -727,7 +727,7 @@ namespace Opm {
}
OPM_END_PARALLEL_TRY_CATCH("BlackoilModelEbos::localConvergenceData() failed: ");
OPM_END_PARALLEL_TRY_CATCH("BlackoilModelEbos::localConvergenceData() failed: ", grid_.comm());
// compute local average in terms of global number of elements
const int bSize = B_avg.size();
@ -772,7 +772,7 @@ namespace Opm {
}
}
OPM_END_PARALLEL_TRY_CATCH("BlackoilModelEbos::ComputeCnvError() failed: ");
OPM_END_PARALLEL_TRY_CATCH("BlackoilModelEbos::ComputeCnvError() failed: ", grid_.comm());
return grid_.comm().sum(errorPV);
}

42
opm/simulators/flow/FlowMainEbos.hpp
View File

@ -25,6 +25,7 @@
#include <sys/utsname.h>
#include <dune/common/version.hh>
#include <opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp>
#include <opm/simulators/utils/ParallelFileMerger.hpp>
#include <opm/simulators/utils/moduleVersion.hpp>
@ -44,6 +45,14 @@
#include <dune/common/parallel/mpihelper.hh>
#endif
namespace Opm::Parallel {
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
}
namespace Opm::Properties {
template<class TypeTag, class MyTypeTag>
@ -104,7 +113,7 @@ namespace Opm
}
// Read the command line parameters. Throws an exception if something goes wrong.
static int setupParameters_(int argc, char** argv)
static int setupParameters_(int argc, char** argv, Parallel::Communication comm)
{
using ParamsMeta = GetProp<TypeTag, Properties::ParameterMetaData>;
if (!ParamsMeta::registrationOpen()) {
@ -214,10 +223,7 @@ namespace Opm
EWOMS_END_PARAM_REGISTRATION(TypeTag);
int mpiRank = 0;
#if HAVE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
#endif
int mpiRank = comm.rank();
// read in the command line parameters
int status = ::Opm::setupParameters_<TypeTag>(argc, const_cast<const char**>(argv), /*doRegistration=*/false, /*allowUnused=*/true, /*handleHelp=*/(mpiRank==0));
@ -229,11 +235,8 @@ namespace Opm
if (mpiRank == 0) {
unknownKeyWords = Parameters::printUnused<TypeTag>(std::cerr);
}
#if HAVE_MPI
int globalUnknownKeyWords;
MPI_Allreduce(&unknownKeyWords, &globalUnknownKeyWords, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
int globalUnknownKeyWords = comm.sum(unknownKeyWords);
unknownKeyWords = globalUnknownKeyWords;
#endif
if ( unknownKeyWords )
{
if ( mpiRank == 0 )
@ -275,7 +278,7 @@ namespace Opm
return status;
}
static void printBanner()
static void printBanner(Parallel::Communication comm)
{
const int lineLen = 70;
const std::string version = moduleVersionName();
@ -293,7 +296,6 @@ namespace Opm
std::cout << "**********************************************************************\n\n";
int threads = 1;
int mpiSize = 1;
#ifdef _OPENMP
// This function is called before the parallel OpenMP stuff gets initialized.
@ -310,9 +312,7 @@ namespace Opm
threads = std::min(input_threads, omp_get_max_threads());
#endif
#if HAVE_MPI
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
#endif
int mpiSize = comm.size();
std::cout << "Using "<< mpiSize << " MPI processes with "<< threads <<" OMP threads on each \n\n";
}
@ -402,7 +402,7 @@ namespace Opm
try {
// deal with some administrative boilerplate
int status = setupParameters_(this->argc_, this->argv_);
int status = setupParameters_(this->argc_, this->argv_, EclGenericVanguard::comm());
if (status)
return status;
@ -450,13 +450,9 @@ namespace Opm
// determine the rank of the current process and the number of processes
// involved in the simulation. MPI must have already been initialized
// here. (yes, the name of this method is misleading.)
#if HAVE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank_);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size_);
#else
mpi_rank_ = 0;
mpi_size_ = 1;
#endif
auto comm = EclGenericVanguard::comm();
mpi_rank_ = comm.rank();
mpi_size_ = comm.size();
#if _OPENMP
// if openMP is available, default to 2 threads per process.
@ -502,7 +498,7 @@ namespace Opm
void setupEbosSimulator()
{
ebosSimulator_.reset(new EbosSimulator(/*verbose=*/false));
ebosSimulator_.reset(new EbosSimulator(EclGenericVanguard::comm(), /*verbose=*/false));
ebosSimulator_->executionTimer().start();
ebosSimulator_->model().applyInitialSolution();

60
opm/simulators/flow/Main.hpp
View File

@ -149,8 +149,27 @@ namespace Opm
initMPI();
}
#define DEMONSTRATE_RUN_WITH_NONWORLD_COMM 1
~Main()
{
#if DEMONSTRATE_RUN_WITH_NONWORLD_COMM
#if HAVE_MPI
// Cannot use EclGenericVanguard::comm()
// to get world size here, as it may be
// a split communication at this point.
int world_size;
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
if (world_size > 1) {
MPI_Comm new_comm = EclGenericVanguard::comm();
int result;
MPI_Comm_compare(MPI_COMM_WORLD, new_comm, &result);
assert(result == MPI_UNEQUAL);
MPI_Comm_free(&new_comm);
}
#endif // HAVE_MPI
#endif // DEMONSTRATE_RUN_WITH_NONWORLD_COMM
EclGenericVanguard::setCommunication(nullptr);
#if HAVE_MPI && !HAVE_DUNE_FEM
@ -175,28 +194,43 @@ namespace Opm
#elif HAVE_MPI
MPI_Init(&argc_, &argv_);
#endif
EclGenericVanguard::setCommunication(std::make_unique<EclGenericVanguard::CommunicationType>());
EclGenericVanguard::setCommunication(std::make_unique<Parallel::Communication>());
#if DEMONSTRATE_RUN_WITH_NONWORLD_COMM
#if HAVE_MPI
if (EclGenericVanguard::comm().size() > 1) {
int world_rank = EclGenericVanguard::comm().rank();
int color = (world_rank == 0);
MPI_Comm new_comm;
MPI_Comm_split(EclGenericVanguard::comm(), color, world_rank, &new_comm);
isSimulationRank_ = (world_rank > 0);
EclGenericVanguard::setCommunication(std::make_unique<Parallel::Communication>(new_comm));
}
#endif // HAVE_MPI
#endif // DEMONSTRATE_RUN_WITH_NONWORLD_COMM
}
int runDynamic()
{
int exitCode = EXIT_SUCCESS;
if (initialize_<Properties::TTag::FlowEarlyBird>(exitCode)) {
return dispatchDynamic_();
} else {
return exitCode;
if (isSimulationRank_) {
if (initialize_<Properties::TTag::FlowEarlyBird>(exitCode)) {
return dispatchDynamic_();
}
}
return exitCode;
}
template <class TypeTag>
int runStatic()
{
int exitCode = EXIT_SUCCESS;
if (initialize_<TypeTag>(exitCode)) {
return dispatchStatic_<TypeTag>();
} else {
return exitCode;
if (isSimulationRank_) {
if (initialize_<TypeTag>(exitCode)) {
return dispatchStatic_<TypeTag>();
}
}
return exitCode;
}
// To be called from the Python interface code. Only do the
@ -388,7 +422,7 @@ namespace Opm
using PreProblem = GetPropType<PreTypeTag, Properties::Problem>;
PreProblem::setBriefDescription("Flow, an advanced reservoir simulator for ECL-decks provided by the Open Porous Media project.");
int status = FlowMainEbos<PreTypeTag>::setupParameters_(argc_, argv_);
int status = FlowMainEbos<PreTypeTag>::setupParameters_(argc_, argv_, EclGenericVanguard::comm());
if (status != 0) {
// if setupParameters_ returns a value smaller than 0, there was no error, but
// the program should abort. This is the case e.g. for the --help and the
@ -433,7 +467,7 @@ namespace Opm
return false;
}
if (outputCout_) {
FlowMainEbos<PreTypeTag>::printBanner();
FlowMainEbos<PreTypeTag>::printBanner(EclGenericVanguard::comm());
}
// Create Deck and EclipseState.
try {
@ -466,7 +500,7 @@ namespace Opm
if (output_param >= 0)
outputInterval = output_param;
readDeck(mpiRank, deckFilename, deck_, eclipseState_, schedule_, udqState_, actionState_,
readDeck(EclGenericVanguard::comm(), deckFilename, deck_, eclipseState_, schedule_, udqState_, actionState_,
summaryConfig_, nullptr, python, std::move(parseContext),
init_from_restart_file, outputCout_, outputInterval);
@ -552,6 +586,8 @@ namespace Opm
std::shared_ptr<EclipseState> eclipseState_;
std::shared_ptr<Schedule> schedule_;
std::shared_ptr<SummaryConfig> summaryConfig_;
// To demonstrate run with non_world_comm
bool isSimulationRank_ = true;
};
} // namespace Opm

13
opm/simulators/linalg/ParallelIstlInformation.hpp
View File

@ -44,6 +44,17 @@
#include <dune/common/enumset.hh>
#include <opm/common/utility/platform_dependent/reenable_warnings.h>
#include <dune/common/version.hh>
#include <dune/common/parallel/mpihelper.hh>
namespace Opm::Parallel {
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
} // end namespace Communication
namespace Opm
{
namespace
@ -109,7 +120,7 @@ public:
return remoteIndices_;
}
/// \brief Get the Collective MPI communicator that we use.
Dune::CollectiveCommunication<MPI_Comm> communicator() const
Parallel::Communication communicator() const
{
return communicator_;
}

2
opm/simulators/linalg/getQuasiImpesWeights.hpp
View File

@ -128,7 +128,7 @@ namespace Amg
weights[index] = bweights;
++index;
}
OPM_END_PARALLEL_TRY_CATCH("getTrueImpesWeights() failed: ");
OPM_END_PARALLEL_TRY_CATCH("getTrueImpesWeights() failed: ", elemCtx.simulator().vanguard().grid().comm());
}
} // namespace Amg

84
opm/simulators/timestepping/gatherConvergenceReport.cpp
View File

@ -33,57 +33,57 @@ namespace
void packReservoirFailure(const ConvergenceReport::ReservoirFailure& f,
std::vector<char>& buf,
int& offset)
int& offset, MPI_Comm mpi_communicator)
{
int type = static_cast<int>(f.type());
int severity = static_cast<int>(f.severity());
int phase = f.phase();
MPI_Pack(&type, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&severity, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&phase, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&type, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
MPI_Pack(&severity, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
MPI_Pack(&phase, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
}
void packWellFailure(const ConvergenceReport::WellFailure& f,
std::vector<char>& buf,
int& offset)
int& offset, MPI_Comm mpi_communicator)
{
int type = static_cast<int>(f.type());
int severity = static_cast<int>(f.severity());
int phase = f.phase();
MPI_Pack(&type, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&severity, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&phase, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&type, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
MPI_Pack(&severity, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
MPI_Pack(&phase, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
int name_length = f.wellName().size() + 1; // Adding 1 for the null terminator.
MPI_Pack(&name_length, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(const_cast<char*>(f.wellName().c_str()), name_length, MPI_CHAR, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&name_length, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
MPI_Pack(const_cast<char*>(f.wellName().c_str()), name_length, MPI_CHAR, buf.data(), buf.size(), &offset, mpi_communicator);
}
void packConvergenceReport(const ConvergenceReport& local_report,
std::vector<char>& buf,
int& offset)
int& offset, MPI_Comm mpi_communicator)
{
// Pack the data.
// Status will not be packed, it is possible to deduce from the other data.
// Reservoir failures.
const auto rf = local_report.reservoirFailures();
int num_rf = rf.size();
MPI_Pack(&num_rf, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&num_rf, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
for (const auto& f : rf) {
packReservoirFailure(f, buf, offset);
packReservoirFailure(f, buf, offset, mpi_communicator);
}
// Well failures.
const auto wf = local_report.wellFailures();
int num_wf = wf.size();
MPI_Pack(&num_wf, 1, MPI_INT, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&num_wf, 1, MPI_INT, buf.data(), buf.size(), &offset, mpi_communicator);
for (const auto& f : wf) {
packWellFailure(f, buf, offset);
packWellFailure(f, buf, offset, mpi_communicator);
}
}
int messageSize(const ConvergenceReport& local_report)
int messageSize(const ConvergenceReport& local_report, MPI_Comm mpi_communicator)
{
int int_pack_size = 0;
MPI_Pack_size(1, MPI_INT, MPI_COMM_WORLD, &int_pack_size);
MPI_Pack_size(1, MPI_INT, mpi_communicator, &int_pack_size);
const int num_rf = local_report.reservoirFailures().size();
const int num_wf = local_report.wellFailures().size();
int wellnames_length = 0;
@ -93,33 +93,33 @@ namespace
return (2 + 3*num_rf + 4*num_wf) * int_pack_size + wellnames_length;
}
ConvergenceReport::ReservoirFailure unpackReservoirFailure(const std::vector<char>& recv_buffer, int& offset)
ConvergenceReport::ReservoirFailure unpackReservoirFailure(const std::vector<char>& recv_buffer, int& offset, MPI_Comm mpi_communicator)
{
int type = -1;
int severity = -1;
int phase = -1;
auto* data = const_cast<char*>(recv_buffer.data());
MPI_Unpack(data, recv_buffer.size(), &offset, &type, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &severity, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &phase, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &type, 1, MPI_INT, mpi_communicator);
MPI_Unpack(data, recv_buffer.size(), &offset, &severity, 1, MPI_INT, mpi_communicator);
MPI_Unpack(data, recv_buffer.size(), &offset, &phase, 1, MPI_INT, mpi_communicator);
return ConvergenceReport::ReservoirFailure(static_cast<ConvergenceReport::ReservoirFailure::Type>(type),
static_cast<ConvergenceReport::Severity>(severity),
phase);
}
ConvergenceReport::WellFailure unpackWellFailure(const std::vector<char>& recv_buffer, int& offset)
ConvergenceReport::WellFailure unpackWellFailure(const std::vector<char>& recv_buffer, int& offset, MPI_Comm mpi_communicator)
{
int type = -1;
int severity = -1;
int phase = -1;
auto* data = const_cast<char*>(recv_buffer.data());
MPI_Unpack(data, recv_buffer.size(), &offset, &type, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &severity, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &phase, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &type, 1, MPI_INT, mpi_communicator);
MPI_Unpack(data, recv_buffer.size(), &offset, &severity, 1, MPI_INT, mpi_communicator);
MPI_Unpack(data, recv_buffer.size(), &offset, &phase, 1, MPI_INT, mpi_communicator);
int name_length = -1;
MPI_Unpack(data, recv_buffer.size(), &offset, &name_length, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &name_length, 1, MPI_INT, mpi_communicator);
std::vector<char> namechars(name_length);
MPI_Unpack(data, recv_buffer.size(), &offset, namechars.data(), name_length, MPI_CHAR, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, namechars.data(), name_length, MPI_CHAR, mpi_communicator);
std::string name(namechars.data());
return ConvergenceReport::WellFailure(static_cast<ConvergenceReport::WellFailure::Type>(type),
static_cast<ConvergenceReport::Severity>(severity),
@ -127,33 +127,33 @@ namespace
name);
}
ConvergenceReport unpackSingleConvergenceReport(const std::vector<char>& recv_buffer, int& offset)
ConvergenceReport unpackSingleConvergenceReport(const std::vector<char>& recv_buffer, int& offset, MPI_Comm mpi_communicator)
{
ConvergenceReport cr;
int num_rf = -1;
auto* data = const_cast<char*>(recv_buffer.data());
MPI_Unpack(data, recv_buffer.size(), &offset, &num_rf, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &num_rf, 1, MPI_INT, mpi_communicator);
for (int rf = 0; rf < num_rf; ++rf) {
ConvergenceReport::ReservoirFailure f = unpackReservoirFailure(recv_buffer, offset);
ConvergenceReport::ReservoirFailure f = unpackReservoirFailure(recv_buffer, offset, mpi_communicator);
cr.setReservoirFailed(f);
}
int num_wf = -1;
MPI_Unpack(data, recv_buffer.size(), &offset, &num_wf, 1, MPI_INT, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &num_wf, 1, MPI_INT, mpi_communicator);
for (int wf = 0; wf < num_wf; ++wf) {
ConvergenceReport::WellFailure f = unpackWellFailure(recv_buffer, offset);
ConvergenceReport::WellFailure f = unpackWellFailure(recv_buffer, offset, mpi_communicator);
cr.setWellFailed(f);
}
return cr;
}
ConvergenceReport unpackConvergenceReports(const std::vector<char>& recv_buffer,
const std::vector<int>& displ)
const std::vector<int>& displ, MPI_Comm mpi_communicator)
{
ConvergenceReport cr;
const int num_processes = displ.size() - 1;
for (int process = 0; process < num_processes; ++process) {
int offset = displ[process];
cr += unpackSingleConvergenceReport(recv_buffer, offset);
cr += unpackSingleConvergenceReport(recv_buffer, offset, mpi_communicator);
assert(offset == displ[process + 1]);
}
return cr;
@ -167,20 +167,20 @@ namespace Opm
/// Create a global convergence report combining local
/// (per-process) reports.
ConvergenceReport gatherConvergenceReport(const ConvergenceReport& local_report)
ConvergenceReport gatherConvergenceReport(const ConvergenceReport& local_report, Parallel::Communication mpi_communicator)
{
// Pack local report.
int message_size = messageSize(local_report);
int message_size = messageSize(local_report, mpi_communicator);
std::vector<char> buffer(message_size);
int offset = 0;
packConvergenceReport(local_report, buffer, offset);
packConvergenceReport(local_report, buffer, offset,mpi_communicator);
assert(offset == message_size);
// Get message sizes and create offset/displacement array for gathering.
int num_processes = -1;
MPI_Comm_size(MPI_COMM_WORLD, &num_processes);
MPI_Comm_size(mpi_communicator, &num_processes);
std::vector<int> message_sizes(num_processes);
MPI_Allgather(&message_size, 1, MPI_INT, message_sizes.data(), 1, MPI_INT, MPI_COMM_WORLD);
MPI_Allgather(&message_size, 1, MPI_INT, message_sizes.data(), 1, MPI_INT, mpi_communicator);
std::vector<int> displ(num_processes + 1, 0);
std::partial_sum(message_sizes.begin(), message_sizes.end(), displ.begin() + 1);
@ -189,10 +189,10 @@ namespace Opm
MPI_Allgatherv(buffer.data(), buffer.size(), MPI_PACKED,
const_cast<char*>(recv_buffer.data()), message_sizes.data(),
displ.data(), MPI_PACKED,
MPI_COMM_WORLD);
mpi_communicator);
// Unpack.
ConvergenceReport global_report = unpackConvergenceReports(recv_buffer, displ);
ConvergenceReport global_report = unpackConvergenceReports(recv_buffer, displ, mpi_communicator);
return global_report;
}
@ -202,7 +202,7 @@ namespace Opm
namespace Opm
{
ConvergenceReport gatherConvergenceReport(const ConvergenceReport& local_report)
ConvergenceReport gatherConvergenceReport(const ConvergenceReport& local_report, Parallel::Communication mpi_communicator)
{
return local_report;
}

12
opm/simulators/timestepping/gatherConvergenceReport.hpp
View File

@ -21,14 +21,24 @@
#ifndef OPM_GATHERCONVERGENCEREPORT_HEADER_INCLUDED
#define OPM_GATHERCONVERGENCEREPORT_HEADER_INCLUDED
#include <dune/common/version.hh>
#include <opm/simulators/timestepping/ConvergenceReport.hpp>
#include <dune/common/parallel/mpihelper.hh>
namespace Opm::Parallel {
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
} // end namespace Communication
namespace Opm
{
/// Create a global convergence report combining local
/// (per-process) reports.
ConvergenceReport gatherConvergenceReport(const ConvergenceReport& local_report);
ConvergenceReport gatherConvergenceReport(const ConvergenceReport& local_report, Parallel::Communication communicator);
} // namespace Opm

14
opm/simulators/utils/DeferredLogger.hpp
View File

@ -21,9 +21,20 @@
#ifndef OPM_DEFERREDLOGGER_HEADER_INCLUDED
#define OPM_DEFERREDLOGGER_HEADER_INCLUDED
#include <dune/common/version.hh>
#include <dune/common/parallel/mpihelper.hh>
#include <string>
#include <vector>
namespace Opm::Parallel {
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
}
namespace Opm
{
/** This class implements a deferred logger:
@ -87,7 +98,8 @@ enum ExcEnum {
private:
std::vector<Message> messages_;
friend DeferredLogger gatherDeferredLogger(const DeferredLogger& local_deferredlogger);
friend DeferredLogger gatherDeferredLogger(const DeferredLogger& local_deferredlogger,
Parallel::Communication mpi_communicator);
};
} // namespace Opm

44
opm/simulators/utils/DeferredLoggingErrorHelpers.hpp
View File

@ -22,6 +22,7 @@
#define OPM_DEFERREDLOGGINGERRORHELPERS_HPP
#include <opm/simulators/utils/DeferredLogger.hpp>
#include <opm/simulators/utils/gatherDeferredLogger.hpp>
#include <opm/material/common/Exceptions.hpp>
@ -53,9 +54,11 @@
namespace {
void _throw(Opm::ExceptionType::ExcEnum exc_type, const std::string& message) {
const auto& cc = Dune::MPIHelper::getCollectiveCommunication();
auto global_exc = cc.max(exc_type);
void _throw(Opm::ExceptionType::ExcEnum exc_type,
const std::string& message,
Opm::Parallel::Communication comm)
{
auto global_exc = comm.max(exc_type);
switch (global_exc) {
case Opm::ExceptionType::NONE:
@ -78,18 +81,25 @@ void _throw(Opm::ExceptionType::ExcEnum exc_type, const std::string& message) {
}
}
} // anonymous namespace
inline void checkForExceptionsAndThrow(Opm::ExceptionType::ExcEnum exc_type,
const std::string& message,
Opm::Parallel::Communication comm)
{
_throw(exc_type, message, comm);
}
inline void checkForExceptionsAndThrow(Opm::ExceptionType::ExcEnum exc_type, const std::string& message)
inline void logAndCheckForExceptionsAndThrow(Opm::DeferredLogger& deferred_logger,
Opm::ExceptionType::ExcEnum exc_type,
const std::string& message,
const bool terminal_output,
Opm::Parallel::Communication comm)
{
_throw(exc_type, message);
}
Opm::DeferredLogger global_deferredLogger = gatherDeferredLogger(deferred_logger, comm);
inline void logAndCheckForExceptionsAndThrow(Opm::DeferredLogger& deferred_logger, Opm::ExceptionType::ExcEnum exc_type , const std::string& message, const bool terminal_output)
{
Opm::DeferredLogger global_deferredLogger = gatherDeferredLogger(deferred_logger);
if (terminal_output) {
global_deferredLogger.logMessages();
}
@ -97,7 +107,7 @@ inline void logAndCheckForExceptionsAndThrow(Opm::DeferredLogger& deferred_logge
// cleared from the global logger, but we must also clear them
// from the local logger.
deferred_logger.clearMessages();
_throw(exc_type, message);
_throw(exc_type, message, comm);
}
@ -139,21 +149,19 @@ catch (const Opm::NumericalIssue& e){ \
///
/// Assumes that OPM_BEGIN_PARALLEL_TRY_CATCH() was called to initiate
/// the try-catch clause
#define OPM_END_PARALLEL_TRY_CATCH(prefix) \
#define OPM_END_PARALLEL_TRY_CATCH(prefix, comm) \
} \
OPM_PARALLEL_CATCH_CLAUSE(obptc_exc_type, obptc_exc_msg);\
checkForExceptionsAndThrow(obptc_exc_type, \
prefix + obptc_exc_msg);
prefix + obptc_exc_msg, comm);
/// \brief Catch exception, log, and throw in a parallel try-catch clause
///
/// Assumes that OPM_BEGIN_PARALLEL_TRY_CATCH() was called to initiate
/// the try-catch clause
#define OPM_END_PARALLEL_TRY_CATCH_LOG(obptc_logger, \
obptc_prefix, \
obptc_output) \
#define OPM_END_PARALLEL_TRY_CATCH_LOG(obptc_logger, obptc_prefix, obptc_output, comm)\
} \
OPM_PARALLEL_CATCH_CLAUSE(obptc_exc_type, obptc_exc_msg); \
logAndCheckForExceptionsAndThrow(obptc_logger, obptc_exc_type, \
obptc_prefix + obptc_exc_msg, obptc_output);
obptc_prefix + obptc_exc_msg, obptc_output, comm);
#endif // OPM_DEFERREDLOGGINGERRORHELPERS_HPP

31
opm/simulators/utils/ParallelEclipseState.cpp
View File

@ -27,7 +27,14 @@ namespace Opm {
ParallelFieldPropsManager::ParallelFieldPropsManager(FieldPropsManager& manager)
: m_manager(manager)
, m_comm(Dune::MPIHelper::getCollectiveCommunication())
, m_comm(Parallel::Communication())
{
}
// EXPERIMENTAL FUNCTION TO ADD COMM AS INPUT
ParallelFieldPropsManager::ParallelFieldPropsManager(FieldPropsManager& manager, Parallel::Communication comm)
: m_manager(manager)
, m_comm(comm)
{
}
@ -208,8 +215,9 @@ bool ParallelFieldPropsManager::has_double(const std::string& keyword) const
}
ParallelEclipseState::ParallelEclipseState()
: m_fieldProps(field_props)
ParallelEclipseState::ParallelEclipseState(Parallel::Communication comm)
: m_fieldProps(field_props, comm)
, m_comm(comm)
{
}
@ -220,13 +228,19 @@ ParallelEclipseState::ParallelEclipseState(const Deck& deck)
{
}
ParallelEclipseState::ParallelEclipseState(const Deck& deck, Parallel::Communication comm)
: EclipseState(deck)
, m_fieldProps(field_props, comm)
, m_comm(comm)
{
}
const FieldPropsManager& ParallelEclipseState::fieldProps() const
{
if (!m_parProps && Dune::MPIHelper::getCollectiveCommunication().rank() != 0)
if (!m_parProps && m_comm.rank() != 0)
OPM_THROW(std::runtime_error, "Attempt to access field properties on no-root process before switch to parallel properties");
if (!m_parProps || Dune::MPIHelper::getCollectiveCommunication().size() == 1)
if (!m_parProps || m_comm.size() == 1)
return this->EclipseState::fieldProps();
return m_fieldProps;
@ -235,7 +249,7 @@ const FieldPropsManager& ParallelEclipseState::fieldProps() const
const FieldPropsManager& ParallelEclipseState::globalFieldProps() const
{
if (Dune::MPIHelper::getCollectiveCommunication().rank() != 0)
if (m_comm.rank() != 0)
OPM_THROW(std::runtime_error, "Attempt to access global field properties on non-root process");
return this->EclipseState::globalFieldProps();
}
@ -243,7 +257,7 @@ const FieldPropsManager& ParallelEclipseState::globalFieldProps() const
const EclipseGrid& ParallelEclipseState::getInputGrid() const
{
if (Dune::MPIHelper::getCollectiveCommunication().rank() != 0)
if (m_comm.rank() != 0)
OPM_THROW(std::runtime_error, "Attempt to access eclipse grid on non-root process");
return this->EclipseState::getInputGrid();
}
@ -257,8 +271,7 @@ void ParallelEclipseState::switchToGlobalProps()
void ParallelEclipseState::switchToDistributedProps()
{
const auto& comm = Dune::MPIHelper::getCollectiveCommunication();
if (comm.size() == 1) // No need for the parallel frontend
if (m_comm.size() == 1) // No need for the parallel frontend
return;
m_parProps = true;

24
opm/simulators/utils/ParallelEclipseState.hpp
View File

@ -19,12 +19,21 @@
#ifndef PARALLEL_ECLIPSE_STATE_HPP
#define PARALLEL_ECLIPSE_STATE_HPP
#include <dune/common/version.hh>
#include <opm/parser/eclipse/EclipseState/EclipseState.hpp>
#include <opm/parser/eclipse/EclipseState/Grid/TranCalculator.hpp>
#include <dune/common/parallel/mpihelper.hh>
#include <functional>
namespace Opm::Parallel {
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
} // end namespace Communication
namespace Opm {
@ -47,6 +56,10 @@ public:
//! \param manager The field property manager to wrap.
ParallelFieldPropsManager(FieldPropsManager& manager);
//! \brief Constructor.
//! \param manager The field property manager to wrap.
ParallelFieldPropsManager(FieldPropsManager& manager, Parallel::Communication comm);
//! \brief Returns actnum vector.
//! \details If called on non-root process an empty vector is returned
std::vector<int> actnum() const override;
@ -106,7 +119,7 @@ protected:
std::map<std::string, Fieldprops::FieldData<int>> m_intProps; //!< Map of integer properties in process-local compressed indices.
std::map<std::string, Fieldprops::FieldData<double>> m_doubleProps; //!< Map of double properties in process-local compressed indices.
FieldPropsManager& m_manager; //!< Underlying field property manager (only used on root process).
Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator> m_comm; //!< Collective communication handler.
Parallel::Communication m_comm; //!< Collective communication handler.
std::function<int(void)> m_activeSize; //!< active size function of the grid
std::function<int(const int)> m_local2Global; //!< mapping from local to global cartesian indices
std::unordered_map<std::string, Fieldprops::TranCalculator> m_tran; //!< calculators map
@ -130,13 +143,19 @@ class ParallelEclipseState : public EclipseState {
friend class PropsCentroidsDataHandle;
public:
//! \brief Default constructor.
ParallelEclipseState();
ParallelEclipseState(Parallel::Communication comm);
//! \brief Construct from a deck instance.
//! \param deck The deck to construct from
//! \details Only called on root process
ParallelEclipseState(const Deck& deck);
//! EXPERIMENTAL FUNCTION TO ADD COMM AS INPUT.
//! \brief Construct from a deck instance.
//! \param deck The deck to construct from
//! \details Only called on root process
ParallelEclipseState(const Deck& deck, Parallel::Communication comm);
//! \brief Switch to global field properties.
//! \details Called on root process to use the global field properties
void switchToGlobalProps();
@ -170,6 +189,7 @@ public:
private:
bool m_parProps = false; //! True to use distributed properties on root process
ParallelFieldPropsManager m_fieldProps; //!< The parallel field properties
Parallel::Communication m_comm; //!< Collective communication handler.
};

200
opm/simulators/utils/ParallelRestart.cpp
View File

@ -39,17 +39,17 @@
#include <opm/parser/eclipse/EclipseState/Schedule/SummaryState.hpp>
#define HANDLE_AS_POD(T) \
std::size_t packSize(const T& data, Dune::MPIHelper::MPICommunicator comm) \
std::size_t packSize(const T& data, Opm::Parallel::MPIComm comm) \
{ \
return packSize(data, comm, std::integral_constant<bool,true>()); \
} \
void pack(const T& data, std::vector<char>& buffer, int& position, \
Dune::MPIHelper::MPICommunicator comm) \
Opm::Parallel::MPIComm comm) \
{ \
pack(data, buffer, position, comm, std::integral_constant<bool,true>()); \
} \
void unpack(T& data, std::vector<char>& buffer, int& position, \
Dune::MPIHelper::MPICommunicator comm) \
Opm::Parallel::MPIComm comm) \
{ \
unpack(data, buffer, position, comm, std::integral_constant<bool,true>()); \
}
@ -59,14 +59,14 @@ namespace Opm
namespace Mpi
{
template<class T>
std::size_t packSize(const T*, std::size_t, Dune::MPIHelper::MPICommunicator,
std::size_t packSize(const T*, std::size_t, Opm::Parallel::MPIComm,
std::integral_constant<bool, false>)
{
OPM_THROW(std::logic_error, "Packing not (yet) supported for this non-pod type.");
}
template<class T>
std::size_t packSize(const T*, std::size_t l, Dune::MPIHelper::MPICommunicator comm,
std::size_t packSize(const T*, std::size_t l, Opm::Parallel::MPIComm comm,
std::integral_constant<bool, true>)
{
#if HAVE_MPI
@ -82,19 +82,19 @@ std::size_t packSize(const T*, std::size_t l, Dune::MPIHelper::MPICommunicator c
}
template<class T>
std::size_t packSize(const T* data, std::size_t l, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const T* data, std::size_t l, Opm::Parallel::MPIComm comm)
{
return packSize(data, l, comm, typename std::is_pod<T>::type());
}
template<class T1, class T2>
std::size_t packSize(const std::pair<T1,T2>& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::pair<T1,T2>& data, Opm::Parallel::MPIComm comm)
{
return packSize(data.first, comm) + packSize(data.second, comm);
}
template<class T>
std::size_t packSize(const std::optional<T>& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::optional<T>& data, Opm::Parallel::MPIComm comm)
{
bool has_value = data.has_value();
std::size_t pack_size = packSize(has_value, comm);
@ -105,7 +105,7 @@ std::size_t packSize(const std::optional<T>& data, Dune::MPIHelper::MPICommunica
template<class T, class A>
std::size_t packSize(const std::vector<T,A>& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::vector<T,A>& data, Opm::Parallel::MPIComm comm)
{
if (std::is_pod<T>::value)
// size written automatically
@ -120,7 +120,7 @@ std::size_t packSize(const std::vector<T,A>& data, Dune::MPIHelper::MPICommunica
}
template<class A>
std::size_t packSize(const std::vector<bool,A>& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::vector<bool,A>& data, Opm::Parallel::MPIComm comm)
{
bool entry;
return packSize(data.size(), comm) + data.size()*packSize(entry,comm);
@ -128,27 +128,27 @@ std::size_t packSize(const std::vector<bool,A>& data, Dune::MPIHelper::MPICommun
template<std::size_t I = 0, typename Tuple>
typename std::enable_if<I == std::tuple_size<Tuple>::value, std::size_t>::type
pack_size_tuple_entry(const Tuple&, Dune::MPIHelper::MPICommunicator)
pack_size_tuple_entry(const Tuple&, Opm::Parallel::MPIComm)
{
return 0;
}
template<std::size_t I = 0, typename Tuple>
typename std::enable_if<I != std::tuple_size<Tuple>::value, std::size_t>::type
pack_size_tuple_entry(const Tuple& tuple, Dune::MPIHelper::MPICommunicator comm)
pack_size_tuple_entry(const Tuple& tuple, Opm::Parallel::MPIComm comm)
{
return packSize(std::get<I>(tuple), comm) + pack_size_tuple_entry<I+1>(tuple, comm);
}
template<class... Ts>
std::size_t packSize(const std::tuple<Ts...>& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::tuple<Ts...>& data, Opm::Parallel::MPIComm comm)
{
return pack_size_tuple_entry(data, comm);
}
template<class T, class H, class KE, class A>
std::size_t packSize(const std::unordered_set<T,H,KE,A>& data,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::size_t totalSize = packSize(data.size(), comm);
for (const auto& entry : data)
@ -160,7 +160,7 @@ std::size_t packSize(const std::unordered_set<T,H,KE,A>& data,
template<class K, class C, class A>
std::size_t packSize(const std::set<K,C,A>& data,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::size_t totalSize = packSize(data.size(), comm);
for (const auto& entry : data)
@ -170,7 +170,7 @@ std::size_t packSize(const std::set<K,C,A>& data,
return totalSize;
}
std::size_t packSize(const char* str, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const char* str, Opm::Parallel::MPIComm comm)
{
#if HAVE_MPI
int size;
@ -185,13 +185,13 @@ std::size_t packSize(const char* str, Dune::MPIHelper::MPICommunicator comm)
#endif
}
std::size_t packSize(const std::string& str, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::string& str, Opm::Parallel::MPIComm comm)
{
return packSize(str.c_str(), comm);
}
template<class T1, class T2, class C, class A>
std::size_t packSize(const std::map<T1,T2,C,A>& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::map<T1,T2,C,A>& data, Opm::Parallel::MPIComm comm)
{
std::size_t totalSize = packSize(data.size(), comm);
for (const auto& entry: data)
@ -202,7 +202,7 @@ std::size_t packSize(const std::map<T1,T2,C,A>& data, Dune::MPIHelper::MPICommun
}
template<class T1, class T2, class H, class P, class A>
std::size_t packSize(const std::unordered_map<T1,T2,H,P,A>& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::unordered_map<T1,T2,H,P,A>& data, Opm::Parallel::MPIComm comm)
{
std::size_t totalSize = packSize(data.size(), comm);
for (const auto& entry: data)
@ -213,7 +213,7 @@ std::size_t packSize(const std::unordered_map<T1,T2,H,P,A>& data, Dune::MPIHelpe
}
template<class T, std::size_t N>
std::size_t packSize(const std::array<T,N>& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const std::array<T,N>& data, Opm::Parallel::MPIComm comm)
{
return N*packSize(data[0], comm);
}
@ -227,12 +227,12 @@ HANDLE_AS_POD(data::NodeData)
HANDLE_AS_POD(data::Rates)
HANDLE_AS_POD(data::Segment)
std::size_t packSize(const data::NumericAquiferData& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::NumericAquiferData& data, Opm::Parallel::MPIComm comm)
{
return packSize(data.initPressure, comm);
}
std::size_t packSize(const data::AquiferData& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::AquiferData& data, Opm::Parallel::MPIComm comm)
{
const auto type = 0ull;
@ -263,7 +263,7 @@ std::size_t packSize(const data::AquiferData& data, Dune::MPIHelper::MPICommunic
return base;
}
std::size_t packSize(const data::GuideRateValue&, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::GuideRateValue&, Opm::Parallel::MPIComm comm)
{
const auto nItem = static_cast<std::size_t>(data::GuideRateValue::Item::NumItems);
@ -271,19 +271,19 @@ std::size_t packSize(const data::GuideRateValue&, Dune::MPIHelper::MPICommunicat
+ packSize(std::array<double, nItem>{}, comm);
}
std::size_t packSize(const data::GroupGuideRates& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::GroupGuideRates& data, Opm::Parallel::MPIComm comm)
{
return packSize(data.production, comm)
+ packSize(data.injection, comm);
}
std::size_t packSize(const data::GroupData& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::GroupData& data, Opm::Parallel::MPIComm comm)
{
return packSize(data.currentControl, comm)
+ packSize(data.guideRates, comm);
}
std::size_t packSize(const data::Well& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::Well& data, Opm::Parallel::MPIComm comm)
{
std::size_t size = packSize(data.rates, comm);
size += packSize(data.bhp, comm) + packSize(data.thp, comm);
@ -296,37 +296,37 @@ std::size_t packSize(const data::Well& data, Dune::MPIHelper::MPICommunicator co
return size;
}
std::size_t packSize(const data::CellData& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::CellData& data, Opm::Parallel::MPIComm comm)
{
return packSize(data.dim, comm) + packSize(data.data, comm) + packSize(data.target, comm);
}
std::size_t packSize(const RestartKey& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const RestartKey& data, Opm::Parallel::MPIComm comm)
{
return packSize(data.key, comm) + packSize(data.dim, comm) + packSize(data.required, comm);
}
std::size_t packSize(const data::Solution& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::Solution& data, Opm::Parallel::MPIComm comm)
{
// Needs explicit conversion to a supported base type holding the data
// to prevent throwing.
return packSize(static_cast<const std::map< std::string, data::CellData>&>(data), comm);
}
std::size_t packSize(const data::GroupAndNetworkValues& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::GroupAndNetworkValues& data, Opm::Parallel::MPIComm comm)
{
return packSize(data.groupData, comm)
+ packSize(data.nodeData, comm);
}
std::size_t packSize(const data::Wells& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const data::Wells& data, Opm::Parallel::MPIComm comm)
{
// Needs explicit conversion to a supported base type holding the data
// to prevent throwing.
return packSize(static_cast<const std::map< std::string, data::Well>&>(data), comm);
}
std::size_t packSize(const RestartValue& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const RestartValue& data, Opm::Parallel::MPIComm comm)
{
return packSize(data.solution, comm)
+ packSize(data.wells, comm)
@ -335,7 +335,7 @@ std::size_t packSize(const RestartValue& data, Dune::MPIHelper::MPICommunicator
+ packSize(data.extra, comm);
}
std::size_t packSize(const Opm::time_point&, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const Opm::time_point&, Opm::Parallel::MPIComm comm)
{
std::time_t tp;
return packSize(tp, comm);
@ -346,14 +346,14 @@ std::size_t packSize(const Opm::time_point&, Dune::MPIHelper::MPICommunicator co
template<class T>
void pack(const T*, std::size_t, std::vector<char>&, int&,
Dune::MPIHelper::MPICommunicator, std::integral_constant<bool, false>)
Opm::Parallel::MPIComm, std::integral_constant<bool, false>)
{
OPM_THROW(std::logic_error, "Packing not (yet) supported for this non-pod type.");
}
template<class T>
void pack(const T* data, std::size_t l, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm,
Opm::Parallel::MPIComm comm,
std::integral_constant<bool, true>)
{
#if HAVE_MPI
@ -372,14 +372,14 @@ void pack(const T* data, std::size_t l, std::vector<char>& buffer, int& position
template<class T>
void pack(const T* data, std::size_t l, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data, l, buffer, position, comm, typename std::is_pod<T>::type());
}
template<class T1, class T2>
void pack(const std::pair<T1,T2>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.first, buffer, position, comm);
pack(data.second, buffer, position, comm);
@ -387,7 +387,7 @@ void pack(const std::pair<T1,T2>& data, std::vector<char>& buffer, int& position
template<class T>
void pack(const std::optional<T>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
bool has_value = data.has_value();
pack(has_value, buffer, position, comm);
@ -398,7 +398,7 @@ void pack(const std::optional<T>& data, std::vector<char>& buffer, int& position
template<class T, class A>
void pack(const std::vector<T, A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
if (std::is_pod<T>::value)
{
@ -416,7 +416,7 @@ void pack(const std::vector<T, A>& data, std::vector<char>& buffer, int& positio
template<class K, class C, class A>
void pack(const std::set<K,C,A>& data,
std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.size(), buffer, position, comm);
@ -429,7 +429,7 @@ void pack(const std::set<K,C,A>& data,
template<class T, class H, class KE, class A>
void pack(const std::unordered_set<T,H,KE,A>& data,
std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.size(), buffer, position, comm);
@ -441,7 +441,7 @@ void pack(const std::unordered_set<T,H,KE,A>& data,
template<class T, size_t N>
void pack(const std::array<T,N>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
for (const T& entry : data)
pack(entry, buffer, position, comm);
@ -449,7 +449,7 @@ void pack(const std::array<T,N>& data, std::vector<char>& buffer, int& position,
template<class A>
void pack(const std::vector<bool,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.size(), buffer, position, comm);
for (const auto entry : data) { // Not a reference: vector<bool> range
@ -461,14 +461,14 @@ void pack(const std::vector<bool,A>& data, std::vector<char>& buffer, int& posit
template<std::size_t I = 0, typename Tuple>
typename std::enable_if<I == std::tuple_size<Tuple>::value, void>::type
pack_tuple_entry(const Tuple&, std::vector<char>&, int&,
Dune::MPIHelper::MPICommunicator)
Opm::Parallel::MPIComm)
{
}
template<std::size_t I = 0, typename Tuple>
typename std::enable_if<I != std::tuple_size<Tuple>::value, void>::type
pack_tuple_entry(const Tuple& tuple, std::vector<char>& buffer,
int& position, Dune::MPIHelper::MPICommunicator comm)
int& position, Opm::Parallel::MPIComm comm)
{
pack(std::get<I>(tuple), buffer, position, comm);
pack_tuple_entry<I+1>(tuple, buffer, position, comm);
@ -476,13 +476,13 @@ pack_tuple_entry(const Tuple& tuple, std::vector<char>& buffer,
template<class... Ts>
void pack(const std::tuple<Ts...>& data, std::vector<char>& buffer,
int& position, Dune::MPIHelper::MPICommunicator comm)
int& position, Opm::Parallel::MPIComm comm)
{
pack_tuple_entry(data, buffer, position, comm);
}
void pack(const char* str, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
#if HAVE_MPI
std::size_t length = strlen(str)+1;
@ -499,14 +499,14 @@ void pack(const char* str, std::vector<char>& buffer, int& position,
}
void pack(const std::string& str, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(str.c_str(), buffer, position, comm);
}
template<class T1, class T2, class C, class A>
void pack(const std::map<T1,T2,C,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.size(), buffer, position, comm);
@ -518,7 +518,7 @@ void pack(const std::map<T1,T2,C,A>& data, std::vector<char>& buffer, int& posit
template<class T1, class T2, class H, class P, class A>
void pack(const std::unordered_map<T1,T2,H,P,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.size(), buffer, position, comm);
@ -529,13 +529,13 @@ void pack(const std::unordered_map<T1,T2,H,P,A>& data, std::vector<char>& buffer
}
void pack(const data::NumericAquiferData& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.initPressure, buffer, position, comm);
}
void pack(const data::AquiferData& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
const auto type =
(data.typeData.is<data::AquiferType::Fetkovich>() * (1ull << 0))
@ -568,7 +568,7 @@ void pack(const data::AquiferData& data, std::vector<char>& buffer, int& positio
}
void pack(const data::GuideRateValue& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
using Item = data::GuideRateValue::Item;
const auto nItem = static_cast<std::size_t>(Item::NumItems);
@ -590,21 +590,21 @@ void pack(const data::GuideRateValue& data, std::vector<char>& buffer, int& posi
}
void pack(const data::GroupGuideRates& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.production, buffer, position, comm);
pack(data.injection, buffer, position, comm);
}
void pack(const data::GroupData& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.currentControl, buffer, position, comm);
pack(data.guideRates, buffer, position, comm);
}
void pack(const data::Well& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.rates, buffer, position, comm);
pack(data.bhp, buffer, position, comm);
@ -618,7 +618,7 @@ void pack(const data::Well& data, std::vector<char>& buffer, int& position,
}
void pack(const RestartKey& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.key, buffer, position, comm);
pack(data.dim, buffer, position, comm);
@ -626,7 +626,7 @@ void pack(const RestartKey& data, std::vector<char>& buffer, int& position,
}
void pack(const data::CellData& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.dim, buffer, position, comm);
pack(data.data, buffer, position, comm);
@ -634,7 +634,7 @@ void pack(const data::CellData& data, std::vector<char>& buffer, int& position,
}
void pack(const data::Solution& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
// Needs explicit conversion to a supported base type holding the data
// to prevent throwing.
@ -643,7 +643,7 @@ void pack(const data::Solution& data, std::vector<char>& buffer, int& position,
}
void pack(const data::Wells& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
// Needs explicit conversion to a supported base type holding the data
// to prevent throwing.
@ -652,14 +652,14 @@ void pack(const data::Wells& data, std::vector<char>& buffer, int& position,
}
void pack(const data::GroupAndNetworkValues& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.groupData, buffer, position, comm);
pack(data.nodeData, buffer, position, comm);
}
void pack(const RestartValue& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(data.solution, buffer, position, comm);
pack(data.wells, buffer, position, comm);
@ -669,7 +669,7 @@ void pack(const RestartValue& data, std::vector<char>& buffer, int& position,
}
void pack(const Opm::time_point& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
pack(Opm::TimeService::to_time_t(data), buffer, position, comm);
}
@ -679,14 +679,14 @@ void pack(const Opm::time_point& data, std::vector<char>& buffer, int& position,
template<class T>
void unpack(T*, const std::size_t&, std::vector<char>&, int&,
Dune::MPIHelper::MPICommunicator, std::integral_constant<bool, false>)
Opm::Parallel::MPIComm, std::integral_constant<bool, false>)
{
OPM_THROW(std::logic_error, "Packing not (yet) supported for this non-pod type.");
}
template<class T>
void unpack(T* data, const std::size_t& l, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm,
Opm::Parallel::MPIComm comm,
std::integral_constant<bool, true>)
{
#if HAVE_MPI
@ -703,14 +703,14 @@ void unpack(T* data, const std::size_t& l, std::vector<char>& buffer, int& posit
template<class T>
void unpack(T* data, const std::size_t& l, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data, l, buffer, position, comm, typename std::is_pod<T>::type());
}
template<class T1, class T2>
void unpack(std::pair<T1,T2>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.first, buffer, position, comm);
unpack(data.second, buffer, position, comm);
@ -718,7 +718,7 @@ void unpack(std::pair<T1,T2>& data, std::vector<char>& buffer, int& position,
template<class T>
void unpack(std::optional<T>&data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
bool has_value;
unpack(has_value, buffer, position, comm);
@ -733,7 +733,7 @@ void unpack(std::optional<T>&data, std::vector<char>& buffer, int& position,
template<class T, class A>
void unpack(std::vector<T,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::size_t length = 0;
unpack(length, buffer, position, comm);
@ -751,7 +751,7 @@ void unpack(std::vector<T,A>& data, std::vector<char>& buffer, int& position,
template<class A>
void unpack(std::vector<bool,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
size_t size;
unpack(size, buffer, position, comm);
@ -767,14 +767,14 @@ void unpack(std::vector<bool,A>& data, std::vector<char>& buffer, int& position,
template<std::size_t I = 0, typename Tuple>
typename std::enable_if<I == std::tuple_size<Tuple>::value, void>::type
unpack_tuple_entry(Tuple&, std::vector<char>&, int&,
Dune::MPIHelper::MPICommunicator)
Opm::Parallel::MPIComm)
{
}
template<std::size_t I = 0, typename Tuple>
typename std::enable_if<I != std::tuple_size<Tuple>::value, void>::type
unpack_tuple_entry(Tuple& tuple, std::vector<char>& buffer,
int& position, Dune::MPIHelper::MPICommunicator comm)
int& position, Opm::Parallel::MPIComm comm)
{
unpack(std::get<I>(tuple), buffer, position, comm);
unpack_tuple_entry<I+1>(tuple, buffer, position, comm);
@ -782,7 +782,7 @@ unpack_tuple_entry(Tuple& tuple, std::vector<char>& buffer,
template<class... Ts>
void unpack(std::tuple<Ts...>& data, std::vector<char>& buffer,
int& position, Dune::MPIHelper::MPICommunicator comm)
int& position, Opm::Parallel::MPIComm comm)
{
unpack_tuple_entry(data, buffer, position, comm);
}
@ -790,7 +790,7 @@ void unpack(std::tuple<Ts...>& data, std::vector<char>& buffer,
template<class K, class C, class A>
void unpack(std::set<K,C,A>& data,
std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::size_t size = 0;
unpack(size, buffer, position, comm);
@ -806,7 +806,7 @@ void unpack(std::set<K,C,A>& data,
template<class T, class H, class KE, class A>
void unpack(std::unordered_set<T,H,KE,A>& data,
std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::size_t size=0;
unpack(size, buffer, position, comm);
@ -821,14 +821,14 @@ void unpack(std::unordered_set<T,H,KE,A>& data,
template<class T, size_t N>
void unpack(std::array<T,N>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
for (T& entry : data)
unpack(entry, buffer, position, comm);
}
void unpack(char* str, std::size_t length, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
#if HAVE_MPI
MPI_Unpack(buffer.data(), buffer.size(), &position, const_cast<char*>(str), length, MPI_CHAR, comm);
@ -842,7 +842,7 @@ void unpack(char* str, std::size_t length, std::vector<char>& buffer, int& posit
}
void unpack(std::string& str, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::size_t length=0;
unpack(length, buffer, position, comm);
@ -854,7 +854,7 @@ void unpack(std::string& str, std::vector<char>& buffer, int& position,
template<class T1, class T2, class C, class A>
void unpack(std::map<T1,T2,C,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::size_t size=0;
unpack(size, buffer, position, comm);
@ -869,7 +869,7 @@ void unpack(std::map<T1,T2,C,A>& data, std::vector<char>& buffer, int& position,
template<class T1, class T2, class H, class P, class A>
void unpack(std::unordered_map<T1,T2,H,P,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::size_t size=0;
unpack(size, buffer, position, comm);
@ -883,7 +883,7 @@ void unpack(std::unordered_map<T1,T2,H,P,A>& data, std::vector<char>& buffer, in
}
void unpack(data::Well& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.rates, buffer, position, comm);
unpack(data.bhp, buffer, position, comm);
@ -897,13 +897,13 @@ void unpack(data::Well& data, std::vector<char>& buffer, int& position,
}
void unpack(data::NumericAquiferData& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.initPressure, buffer, position, comm);
}
void unpack(data::AquiferData& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
auto type = 0ull;
@ -930,7 +930,7 @@ void unpack(data::AquiferData& data, std::vector<char>& buffer, int& position,
}
void unpack(data::GuideRateValue& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
using Item = data::GuideRateValue::Item;
const auto nItem = static_cast<std::size_t>(Item::NumItems);
@ -949,21 +949,21 @@ void unpack(data::GuideRateValue& data, std::vector<char>& buffer, int& position
}
void unpack(data::GroupGuideRates& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.production, buffer, position, comm);
unpack(data.injection, buffer, position, comm);
}
void unpack(data::GroupData& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.currentControl, buffer, position, comm);
unpack(data.guideRates, buffer, position, comm);
}
void unpack(RestartKey& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.key, buffer, position, comm);
unpack(data.dim, buffer, position, comm);
@ -971,7 +971,7 @@ void unpack(RestartKey& data, std::vector<char>& buffer, int& position,
}
void unpack(data::CellData& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.dim, buffer, position, comm);
unpack(data.data, buffer, position, comm);
@ -979,7 +979,7 @@ void unpack(data::CellData& data, std::vector<char>& buffer, int& position,
}
void unpack(data::Solution& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
// Needs explicit conversion to a supported base type holding the data
// to prevent throwing.
@ -988,7 +988,7 @@ void unpack(data::Solution& data, std::vector<char>& buffer, int& position,
}
void unpack(data::Wells& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
// Needs explicit conversion to a supported base type holding the data
// to prevent throwing.
@ -997,14 +997,14 @@ void unpack(data::Wells& data, std::vector<char>& buffer, int& position,
}
void unpack(data::GroupAndNetworkValues& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.groupData, buffer, position, comm);
unpack(data.nodeData, buffer, position, comm);
}
void unpack(RestartValue& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
unpack(data.solution, buffer, position, comm);
unpack(data.wells, buffer, position, comm);
@ -1014,7 +1014,7 @@ void unpack(RestartValue& data, std::vector<char>& buffer, int& position,
}
void unpack([[maybe_unused]] Opm::time_point& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Opm::Parallel::MPIComm comm)
{
std::time_t tp;
unpack(tp, buffer, position, comm);
@ -1026,13 +1026,13 @@ void unpack([[maybe_unused]] Opm::time_point& data, std::vector<char>& buffer, i
#define INSTANTIATE_PACK_VECTOR(...) \
template std::size_t packSize(const std::vector<__VA_ARGS__>& data, \
Dune::MPIHelper::MPICommunicator comm); \
Opm::Parallel::MPIComm comm); \
template void pack(const std::vector<__VA_ARGS__>& data, \
std::vector<char>& buffer, int& position, \
Dune::MPIHelper::MPICommunicator comm); \
Opm::Parallel::MPIComm comm); \
template void unpack(std::vector<__VA_ARGS__>& data, \
std::vector<char>& buffer, int& position, \
Dune::MPIHelper::MPICommunicator comm);
Opm::Parallel::MPIComm comm);
INSTANTIATE_PACK_VECTOR(float)
INSTANTIATE_PACK_VECTOR(double)
@ -1059,13 +1059,13 @@ INSTANTIATE_PACK_VECTOR(std::string)
#define INSTANTIATE_PACK(...) \
template std::size_t packSize(const __VA_ARGS__& data, \
Dune::MPIHelper::MPICommunicator comm); \
Opm::Parallel::MPIComm comm); \
template void pack(const __VA_ARGS__& data, \
std::vector<char>& buffer, int& position, \
Dune::MPIHelper::MPICommunicator comm); \
Opm::Parallel::MPIComm comm); \
template void unpack(__VA_ARGS__& data, \
std::vector<char>& buffer, int& position, \
Dune::MPIHelper::MPICommunicator comm);
Opm::Parallel::MPIComm comm);
INSTANTIATE_PACK(float)
INSTANTIATE_PACK(double)
@ -1101,7 +1101,7 @@ INSTANTIATE_PACK(std::set<std::string>)
RestartValue loadParallelRestart(const EclipseIO* eclIO, Action::State& actionState, SummaryState& summaryState,
const std::vector<Opm::RestartKey>& solutionKeys,
const std::vector<Opm::RestartKey>& extraKeys,
Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator> comm)
Parallel::Communication comm)
{
#if HAVE_MPI
RestartValue restartValues{};

120
opm/simulators/utils/ParallelRestart.hpp
View File

@ -26,6 +26,7 @@
#include <opm/common/ErrorMacros.hpp>
#include <opm/common/utility/TimeService.hpp>
#include <dune/common/version.hh>
#include <dune/common/parallel/mpihelper.hh>
#include <chrono>
@ -39,6 +40,15 @@
#include <unordered_set>
#include <vector>
namespace Opm::Parallel {
using MPIComm = typename Dune::MPIHelper::MPICommunicator;
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<MPIComm>;
#else
using Communication = Dune::CollectiveCommunication<MPIComm>;
#endif
} // end namespace Communication
namespace Opm
{
@ -77,18 +87,18 @@ class State;
namespace Mpi
{
template<class T>
std::size_t packSize(const T*, std::size_t, Dune::MPIHelper::MPICommunicator,
std::size_t packSize(const T*, std::size_t, Parallel::MPIComm,
std::integral_constant<bool, false>);
template<class T>
std::size_t packSize(const T*, std::size_t l, Dune::MPIHelper::MPICommunicator comm,
std::size_t packSize(const T*, std::size_t l, Parallel::MPIComm comm,
std::integral_constant<bool, true>);
template<class T>
std::size_t packSize(const T* data, std::size_t l, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const T* data, std::size_t l, Parallel::MPIComm comm);
template<class T>
std::size_t packSize(const T&, Dune::MPIHelper::MPICommunicator,
std::size_t packSize(const T&, Parallel::MPIComm,
std::integral_constant<bool, false>)
{
std::string msg = std::string{"Packing not (yet) supported for non-pod type: "} + typeid(T).name();
@ -96,7 +106,7 @@ std::size_t packSize(const T&, Dune::MPIHelper::MPICommunicator,
}
template<class T>
std::size_t packSize(const T&, Dune::MPIHelper::MPICommunicator comm,
std::size_t packSize(const T&, Parallel::MPIComm comm,
std::integral_constant<bool, true>)
{
#if HAVE_MPI
@ -110,69 +120,69 @@ std::size_t packSize(const T&, Dune::MPIHelper::MPICommunicator comm,
}
template<class T>
std::size_t packSize(const T& data, Dune::MPIHelper::MPICommunicator comm)
std::size_t packSize(const T& data, Parallel::MPIComm comm)
{
return packSize(data, comm, typename std::is_pod<T>::type());
}
template<class T1, class T2>
std::size_t packSize(const std::pair<T1,T2>& data, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const std::pair<T1,T2>& data, Parallel::MPIComm comm);
template<class T>
std::size_t packSize(const std::optional<T>& data, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const std::optional<T>& data, Parallel::MPIComm comm);
template<class T, class A>
std::size_t packSize(const std::vector<T,A>& data, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const std::vector<T,A>& data, Parallel::MPIComm comm);
template<class K, class C, class A>
std::size_t packSize(const std::set<K,C,A>& data,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T, class H, class KE, class A>
std::size_t packSize(const std::unordered_set<T,H,KE,A>& data,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class A>
std::size_t packSize(const std::vector<bool,A>& data, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const std::vector<bool,A>& data, Parallel::MPIComm comm);
template<class... Ts>
std::size_t packSize(const std::tuple<Ts...>& data, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const std::tuple<Ts...>& data, Parallel::MPIComm comm);
template<class T, std::size_t N>
std::size_t packSize(const std::array<T,N>& data, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const std::array<T,N>& data, Parallel::MPIComm comm);
std::size_t packSize(const char* str, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const char* str, Parallel::MPIComm comm);
template<class T1, class T2, class C, class A>
std::size_t packSize(const std::map<T1,T2,C,A>& data, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const std::map<T1,T2,C,A>& data, Parallel::MPIComm comm);
template<class T1, class T2, class H, class P, class A>
std::size_t packSize(const std::unordered_map<T1,T2,H,P,A>& data, Dune::MPIHelper::MPICommunicator comm);
std::size_t packSize(const std::unordered_map<T1,T2,H,P,A>& data, Parallel::MPIComm comm);
////// pack routines
template<class T>
void pack(const T*, std::size_t, std::vector<char>&, int&,
Dune::MPIHelper::MPICommunicator, std::integral_constant<bool, false>);
Parallel::MPIComm, std::integral_constant<bool, false>);
template<class T>
void pack(const T* data, std::size_t l, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm, std::integral_constant<bool, true>);
Parallel::MPIComm comm, std::integral_constant<bool, true>);
template<class T>
void pack(const T* data, std::size_t l, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T>
void pack(const T&, std::vector<char>&, int&,
Dune::MPIHelper::MPICommunicator, std::integral_constant<bool, false>)
Parallel::MPIComm, std::integral_constant<bool, false>)
{
OPM_THROW(std::logic_error, "Packing not (yet) supported for this non-pod type.");
}
template<class T>
void pack(const T& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm, std::integral_constant<bool, true>)
Parallel::MPIComm comm, std::integral_constant<bool, true>)
{
#if HAVE_MPI
MPI_Pack(&data, 1, Dune::MPITraits<T>::getType(), buffer.data(),
@ -187,81 +197,81 @@ void pack(const T& data, std::vector<char>& buffer, int& position,
template<class T>
void pack(const T& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Parallel::MPIComm comm)
{
pack(data, buffer, position, comm, typename std::is_pod<T>::type());
}
template<class T1, class T2>
void pack(const std::pair<T1,T2>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T>
void pack(const std::optional<T>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T, class A>
void pack(const std::vector<T,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class A>
void pack(const std::vector<bool,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class... Ts>
void pack(const std::tuple<Ts...>& data, std::vector<char>& buffer,
int& position, Dune::MPIHelper::MPICommunicator comm);
int& position, Parallel::MPIComm comm);
template<class K, class C, class A>
void pack(const std::set<K,C,A>& data,
std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T, class H, class KE, class A>
void pack(const std::unordered_set<T,H,KE,A>& data,
std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T, size_t N>
void pack(const std::array<T,N>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T1, class T2, class C, class A>
void pack(const std::map<T1,T2,C,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T1, class T2, class H, class P, class A>
void pack(const std::unordered_map<T1,T2,H,P,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
void pack(const char* str, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
/// unpack routines
template<class T>
void unpack(T*, const std::size_t&, std::vector<char>&, int&,
Dune::MPIHelper::MPICommunicator, std::integral_constant<bool, false>);
Parallel::MPIComm, std::integral_constant<bool, false>);
template<class T>
void unpack(T* data, const std::size_t& l, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm,
Parallel::MPIComm comm,
std::integral_constant<bool, true>);
template<class T>
void unpack(T* data, const std::size_t& l, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T>
void unpack(T&, std::vector<char>&, int&,
Dune::MPIHelper::MPICommunicator, std::integral_constant<bool, false>)
Parallel::MPIComm, std::integral_constant<bool, false>)
{
OPM_THROW(std::logic_error, "Packing not (yet) supported for this non-pod type.");
}
template<class T>
void unpack(T& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm, std::integral_constant<bool, true>)
Parallel::MPIComm comm, std::integral_constant<bool, true>)
{
#if HAVE_MPI
MPI_Unpack(buffer.data(), buffer.size(), &position, &data, 1,
@ -276,64 +286,64 @@ void unpack(T& data, std::vector<char>& buffer, int& position,
template<class T>
void unpack(T& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm)
Parallel::MPIComm comm)
{
unpack(data, buffer, position, comm, typename std::is_pod<T>::type());
}
template<class T1, class T2>
void unpack(std::pair<T1,T2>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T>
void unpack(std::optional<T>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T, class A>
void unpack(std::vector<T,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class A>
void unpack(std::vector<bool,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class... Ts>
void unpack(std::tuple<Ts...>& data, std::vector<char>& buffer,
int& position, Dune::MPIHelper::MPICommunicator comm);
int& position, Parallel::MPIComm comm);
template<class K, class C, class A>
void unpack(std::set<K,C,A>& data,
std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T, class H, class KE, class A>
void unpack(std::unordered_set<T,H,KE,A>& data,
std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T, size_t N>
void unpack(std::array<T,N>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T1, class T2, class C, class A>
void unpack(std::map<T1,T2,C,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
template<class T1, class T2, class H, class P, class A>
void unpack(std::unordered_map<T1,T2,H,P,A>& data, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
void unpack(char* str, std::size_t length, std::vector<char>& buffer, int& position,
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
/// prototypes for complex types
#define ADD_PACK_PROTOTYPES(T) \
std::size_t packSize(const T& data, Dune::MPIHelper::MPICommunicator comm); \
std::size_t packSize(const T& data, Parallel::MPIComm comm); \
void pack(const T& data, std::vector<char>& buffer, int& position, \
Dune::MPIHelper::MPICommunicator comm); \
Parallel::MPIComm comm); \
void unpack(T& data, std::vector<char>& buffer, int& position, \
Dune::MPIHelper::MPICommunicator comm);
Parallel::MPIComm comm);
ADD_PACK_PROTOTYPES(data::AquiferData)
ADD_PACK_PROTOTYPES(data::CarterTracyData)
@ -363,7 +373,7 @@ ADD_PACK_PROTOTYPES(time_point)
RestartValue loadParallelRestart(const EclipseIO* eclIO, Action::State& actionState, SummaryState& summaryState,
const std::vector<RestartKey>& solutionKeys,
const std::vector<RestartKey>& extraKeys,
Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator> comm);
Parallel::Communication comm);
} // end namespace Opm
#endif // PARALLEL_RESTART_HPP

9
opm/simulators/utils/ParallelSerialization.cpp
View File

@ -40,12 +40,13 @@
namespace Opm {
void eclStateBroadcast(EclipseState& eclState, Schedule& schedule,
void eclStateBroadcast(Parallel::Communication comm, EclipseState& eclState, Schedule& schedule,
SummaryConfig& summaryConfig,
UDQState& udqState,
Action::State& actionState)
{
Opm::EclMpiSerializer ser(Dune::MPIHelper::getCollectiveCommunication());
Opm::EclMpiSerializer ser(comm);
ser.broadcast(eclState);
ser.broadcast(schedule);
ser.broadcast(summaryConfig);
@ -53,9 +54,9 @@ void eclStateBroadcast(EclipseState& eclState, Schedule& schedule,
ser.broadcast(actionState);
}
void eclScheduleBroadcast(Schedule& schedule)
void eclScheduleBroadcast(Parallel::Communication comm, Schedule& schedule)
{
Opm::EclMpiSerializer ser(Dune::MPIHelper::getCollectiveCommunication());
Opm::EclMpiSerializer ser(comm);
ser.broadcast(schedule);
}
}

15
opm/simulators/utils/ParallelSerialization.hpp
View File

@ -19,6 +19,17 @@
#ifndef PARALLEL_SERIALIZATION_HPP
#define PARALLEL_SERIALIZATION_HPP
#include <dune/common/version.hh>
#include <dune/common/parallel/mpihelper.hh>
namespace Opm::Parallel {
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
}
namespace Opm {
class EclipseState;
@ -36,13 +47,13 @@ class State;
*! \param schedule Schedule to broadcast
*! \param summaryConfig SummaryConfig to broadcast
*/
void eclStateBroadcast(EclipseState& eclState, Schedule& schedule,
void eclStateBroadcast(Parallel::Communication comm, EclipseState& eclState, Schedule& schedule,
SummaryConfig& summaryConfig,
UDQState& udqState,
Action::State& actionState);
/// \brief Broadcasts an schedule from root node in parallel runs.
void eclScheduleBroadcast(Schedule& schedule);
void eclScheduleBroadcast(Parallel::Communication comm, Schedule& schedule);
} // end namespace Opm

2
opm/simulators/utils/PropsCentroidsDataHandle.hpp
View File

@ -69,7 +69,7 @@ public:
m_centroids(centroids)
{
// Scatter the keys
const auto& comm = Dune::MPIHelper::getCollectiveCommunication();
const Parallel::Communication comm = m_grid.comm();
if (comm.rank() == 0)
{
const auto& globalProps = eclState.globalFieldProps();

56
opm/simulators/utils/gatherDeferredLogger.cpp
View File

@ -32,55 +32,55 @@
namespace
{
void packMessages(const std::vector<Opm::DeferredLogger::Message>& local_messages, std::vector<char>& buf, int& offset)
void packMessages(const std::vector<Opm::DeferredLogger::Message>& local_messages, std::vector<char>& buf, int& offset, const Opm::Parallel::Communication mpi_communicator)
{
int messagesize = local_messages.size();
MPI_Pack(&messagesize, 1, MPI_UNSIGNED, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&messagesize, 1, MPI_UNSIGNED, buf.data(), buf.size(), &offset, mpi_communicator);
for (const auto& lm : local_messages) {
MPI_Pack(static_cast<void*>(const_cast<std::int64_t*>(&lm.flag)), 1, MPI_INT64_T, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(static_cast<void*>(const_cast<std::int64_t*>(&lm.flag)), 1, MPI_INT64_T, buf.data(), buf.size(), &offset, mpi_communicator);
int tagsize = lm.tag.size();
MPI_Pack(&tagsize, 1, MPI_UNSIGNED, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&tagsize, 1, MPI_UNSIGNED, buf.data(), buf.size(), &offset, mpi_communicator);
if (tagsize>0) {
MPI_Pack(const_cast<char*>(lm.tag.c_str()), lm.tag.size(), MPI_CHAR, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(const_cast<char*>(lm.tag.c_str()), lm.tag.size(), MPI_CHAR, buf.data(), buf.size(), &offset, mpi_communicator);
}
int textsize = lm.text.size();
MPI_Pack(&textsize, 1, MPI_UNSIGNED, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(&textsize, 1, MPI_UNSIGNED, buf.data(), buf.size(), &offset, mpi_communicator);
if (textsize>0) {
MPI_Pack(const_cast<char*>(lm.text.c_str()), lm.text.size(), MPI_CHAR, buf.data(), buf.size(), &offset, MPI_COMM_WORLD);
MPI_Pack(const_cast<char*>(lm.text.c_str()), lm.text.size(), MPI_CHAR, buf.data(), buf.size(), &offset, mpi_communicator);
}
}
}
Opm::DeferredLogger::Message unpackSingleMessage(const std::vector<char>& recv_buffer, int& offset)
Opm::DeferredLogger::Message unpackSingleMessage(const std::vector<char>& recv_buffer, int& offset, const MPI_Comm mpi_communicator)
{
int64_t flag;
auto* data = const_cast<char*>(recv_buffer.data());
MPI_Unpack(data, recv_buffer.size(), &offset, &flag, 1, MPI_INT64_T, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &flag, 1, MPI_INT64_T, mpi_communicator);
// unpack tag
unsigned int tagsize;
MPI_Unpack(data, recv_buffer.size(), &offset, &tagsize, 1, MPI_UNSIGNED, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &tagsize, 1, MPI_UNSIGNED, mpi_communicator);
std::string tag;
if (tagsize>0) {
std::vector<char> tagchars(tagsize);
MPI_Unpack(data, recv_buffer.size(), &offset, tagchars.data(), tagsize, MPI_CHAR, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, tagchars.data(), tagsize, MPI_CHAR, mpi_communicator);
tag = std::string(tagchars.data(), tagsize);
}
// unpack text
unsigned int textsize;
MPI_Unpack(data, recv_buffer.size(), &offset, &textsize, 1, MPI_UNSIGNED, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &textsize, 1, MPI_UNSIGNED, mpi_communicator);
std::string text;
if (textsize>0) {
std::vector<char> textchars(textsize);
MPI_Unpack(data, recv_buffer.size(), &offset, textchars.data(), textsize, MPI_CHAR, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, textchars.data(), textsize, MPI_CHAR, mpi_communicator);
text = std::string (textchars.data(), textsize);
}
return Opm::DeferredLogger::Message({flag, tag, text});
}
std::vector<Opm::DeferredLogger::Message> unpackMessages(const std::vector<char>& recv_buffer, const std::vector<int>& displ)
std::vector<Opm::DeferredLogger::Message> unpackMessages(const std::vector<char>& recv_buffer, const std::vector<int>& displ, const MPI_Comm mpi_communicator)
{
std::vector<Opm::DeferredLogger::Message> messages;
const int num_processes = displ.size() - 1;
@ -89,9 +89,9 @@ namespace
int offset = displ[process];
// unpack number of messages
unsigned int messagesize;
MPI_Unpack(data, recv_buffer.size(), &offset, &messagesize, 1, MPI_UNSIGNED, MPI_COMM_WORLD);
MPI_Unpack(data, recv_buffer.size(), &offset, &messagesize, 1, MPI_UNSIGNED, mpi_communicator);
for (unsigned int i=0; i<messagesize; i++) {
messages.push_back(unpackSingleMessage(recv_buffer, offset));
messages.push_back(unpackSingleMessage(recv_buffer, offset, mpi_communicator));
}
assert(offset == displ[process + 1]);
}
@ -105,15 +105,16 @@ namespace Opm
{
/// combine (per-process) messages
Opm::DeferredLogger gatherDeferredLogger(const Opm::DeferredLogger& local_deferredlogger)
Opm::DeferredLogger gatherDeferredLogger(const Opm::DeferredLogger& local_deferredlogger,
Opm::Parallel::Communication mpi_communicator)
{
int num_messages = local_deferredlogger.messages_.size();
int int64_mpi_pack_size;
MPI_Pack_size(1, MPI_INT64_T, MPI_COMM_WORLD, &int64_mpi_pack_size);
MPI_Pack_size(1, MPI_INT64_T, mpi_communicator, &int64_mpi_pack_size);
int unsigned_int_mpi_pack_size;
MPI_Pack_size(1, MPI_UNSIGNED, MPI_COMM_WORLD, &unsigned_int_mpi_pack_size);
MPI_Pack_size(1, MPI_UNSIGNED, mpi_communicator, &unsigned_int_mpi_pack_size);
// store number of messages;
int message_size = unsigned_int_mpi_pack_size;
@ -124,9 +125,9 @@ namespace Opm
for (const auto& lm : local_deferredlogger.messages_) {
int string_mpi_pack_size;
MPI_Pack_size(lm.tag.size(), MPI_CHAR, MPI_COMM_WORLD, &string_mpi_pack_size);
MPI_Pack_size(lm.tag.size(), MPI_CHAR, mpi_communicator, &string_mpi_pack_size);
message_size += string_mpi_pack_size;
MPI_Pack_size(lm.text.size(), MPI_CHAR, MPI_COMM_WORLD, &string_mpi_pack_size);
MPI_Pack_size(lm.text.size(), MPI_CHAR, mpi_communicator, &string_mpi_pack_size);
message_size += string_mpi_pack_size;
}
@ -134,14 +135,14 @@ namespace Opm
std::vector<char> buffer(message_size);
int offset = 0;
packMessages(local_deferredlogger.messages_, buffer, offset);
packMessages(local_deferredlogger.messages_, buffer, offset, mpi_communicator);
assert(offset == message_size);
// Get message sizes and create offset/displacement array for gathering.
int num_processes = -1;
MPI_Comm_size(MPI_COMM_WORLD, &num_processes);
MPI_Comm_size(mpi_communicator, &num_processes);
std::vector<int> message_sizes(num_processes);
MPI_Allgather(&message_size, 1, MPI_INT, message_sizes.data(), 1, MPI_INT, MPI_COMM_WORLD);
MPI_Allgather(&message_size, 1, MPI_INT, message_sizes.data(), 1, MPI_INT, mpi_communicator);
std::vector<int> displ(num_processes + 1, 0);
std::partial_sum(message_sizes.begin(), message_sizes.end(), displ.begin() + 1);
@ -150,11 +151,11 @@ namespace Opm
MPI_Allgatherv(buffer.data(), buffer.size(), MPI_PACKED,
const_cast<char*>(recv_buffer.data()), message_sizes.data(),
displ.data(), MPI_PACKED,
MPI_COMM_WORLD);
mpi_communicator);
// Unpack.
Opm::DeferredLogger global_deferredlogger;
global_deferredlogger.messages_ = unpackMessages(recv_buffer, displ);
global_deferredlogger.messages_ = unpackMessages(recv_buffer, displ, mpi_communicator);
return global_deferredlogger;
}
@ -164,7 +165,8 @@ namespace Opm
namespace Opm
{
Opm::DeferredLogger gatherDeferredLogger(const Opm::DeferredLogger& local_deferredlogger)
Opm::DeferredLogger gatherDeferredLogger(const Opm::DeferredLogger& local_deferredlogger,
Opm::Parallel::Communication /* dummy communicator */)
{
return local_deferredlogger;
}

4
opm/simulators/utils/gatherDeferredLogger.hpp
View File

@ -21,13 +21,15 @@
#ifndef OPM_GATHERDEFERREDLOGGER_HEADER_INCLUDED
#define OPM_GATHERDEFERREDLOGGER_HEADER_INCLUDED
#include <dune/common/parallel/mpihelper.hh>
#include <opm/simulators/utils/DeferredLogger.hpp>
#include <dune/common/version.hh>
namespace Opm
{
/// Create a global log combining local logs
Opm::DeferredLogger gatherDeferredLogger(const Opm::DeferredLogger& local_deferredlogger);
Opm::DeferredLogger gatherDeferredLogger(const Opm::DeferredLogger& local_deferredlogger, Parallel::Communication communicator);
} // namespace Opm

13
opm/simulators/utils/readDeck.cpp
View File

@ -191,7 +191,7 @@ void setupMessageLimiter(const Opm::MessageLimits msgLimits, const std::string&
}
void readDeck(int rank, std::string& deckFilename, std::shared_ptr<Opm::Deck>& deck, std::shared_ptr<Opm::EclipseState>& eclipseState,
void readDeck(Opm::Parallel::Communication comm, std::string& deckFilename, std::shared_ptr<Opm::Deck>& deck, std::shared_ptr<Opm::EclipseState>& eclipseState,
std::shared_ptr<Opm::Schedule>& schedule, std::unique_ptr<UDQState>& udqState, std::unique_ptr<Action::State>& actionState, std::shared_ptr<Opm::SummaryConfig>& summaryConfig,
std::unique_ptr<ErrorGuard> errorGuard, std::shared_ptr<Opm::Python>& python, std::unique_ptr<ParseContext> parseContext,
bool initFromRestart, bool checkDeck, const std::optional<int>& outputInterval)
@ -203,6 +203,7 @@ void readDeck(int rank, std::string& deckFilename, std::shared_ptr<Opm::Deck>& d
int parseSuccess = 1; // > 0 is success
std::string failureMessage;
int rank = comm.rank();
if (rank==0) {
try
@ -216,6 +217,7 @@ void readDeck(int rank, std::string& deckFilename, std::shared_ptr<Opm::Deck>& d
if (!deck)
{
deck = std::make_unique<Opm::Deck>( parser.parseFile(deckFilename , *parseContext, *errorGuard));
Opm::KeywordValidation::KeywordValidator keyword_validator(
@ -233,7 +235,7 @@ void readDeck(int rank, std::string& deckFilename, std::shared_ptr<Opm::Deck>& d
if (!eclipseState) {
#if HAVE_MPI
eclipseState = std::make_unique<Opm::ParallelEclipseState>(*deck);
eclipseState = std::make_unique<Opm::ParallelEclipseState>(*deck, comm);
#else
eclipseState = std::make_unique<Opm::EclipseState>(*deck);
#endif
@ -324,7 +326,7 @@ void readDeck(int rank, std::string& deckFilename, std::shared_ptr<Opm::Deck>& d
if (!schedule)
schedule = std::make_unique<Opm::Schedule>(python);
if (!eclipseState)
eclipseState = std::make_unique<Opm::ParallelEclipseState>();
eclipseState = std::make_unique<Opm::ParallelEclipseState>(comm);
if (!udqState)
udqState = std::make_unique<UDQState>(0);
if (!actionState)
@ -334,13 +336,12 @@ void readDeck(int rank, std::string& deckFilename, std::shared_ptr<Opm::Deck>& d
// In case of parse errors eclipseState/schedule might be null
// and trigger segmentation faults in parallel during broadcast
// (e.g. when serializing the non-existent TableManager)
auto comm = Dune::MPIHelper::getCollectiveCommunication();
parseSuccess = comm.min(parseSuccess);
try
{
if (parseSuccess)
{
Opm::eclStateBroadcast(*eclipseState, *schedule, *summaryConfig, *udqState, *actionState);
Opm::eclStateBroadcast(comm, *eclipseState, *schedule, *summaryConfig, *udqState, *actionState);
}
}
catch(const std::exception& broadcast_error)
@ -359,7 +360,7 @@ void readDeck(int rank, std::string& deckFilename, std::shared_ptr<Opm::Deck>& d
errorGuard->clear();
}
parseSuccess = Dune::MPIHelper::getCollectiveCommunication().min(parseSuccess);
parseSuccess = comm.min(parseSuccess);
if (!parseSuccess)
{

12
opm/simulators/utils/readDeck.hpp
View File

@ -22,6 +22,8 @@
#ifndef OPM_READDECK_HEADER_INCLUDED
#define OPM_READDECK_HEADER_INCLUDED
#include <dune/common/version.hh>
#include <dune/common/parallel/mpihelper.hh>
#include <memory>
#include <optional>
#include <string>
@ -42,6 +44,14 @@ namespace Action {
class State;
}
namespace Parallel {
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
}
enum class FileOutputMode {
//! \brief No output to files.
OUTPUT_NONE = 0,
@ -57,7 +67,7 @@ FileOutputMode setupLogging(int mpi_rank_, const std::string& deck_filename, con
/// \brief Reads the deck and creates all necessary objects if needed
///
/// If pointers already contains objects then they are used otherwise they are created and can be used outside later.
void readDeck(int rank, std::string& deckFilename, std::shared_ptr<Deck>& deck, std::shared_ptr<EclipseState>& eclipseState,
void readDeck(Parallel::Communication comm, std::string& deckFilename, std::shared_ptr<Deck>& deck, std::shared_ptr<EclipseState>& eclipseState,
std::shared_ptr<Schedule>& schedule, std::unique_ptr<UDQState>& udqState, std::unique_ptr<Action::State>& actionState, std::shared_ptr<SummaryConfig>& summaryConfig,
std::unique_ptr<ErrorGuard> errorGuard, std::shared_ptr<Python>& python, std::unique_ptr<ParseContext> parseContext,
bool initFromRestart, bool checkDeck, const std::optional<int>& outputInterval);

9
opm/simulators/wells/BlackoilWellModelGeneric.cpp
View File

@ -49,7 +49,7 @@ BlackoilWellModelGeneric(Schedule& schedule,
const SummaryState& summaryState,
const EclipseState& eclState,
const PhaseUsage& phase_usage,
const Comm& comm)
const Parallel::Communication& comm)
: schedule_(schedule)
, summaryState_(summaryState)
, eclState_(eclState)
@ -1028,7 +1028,7 @@ actionOnBrokenConstraints(const Group& group,
throw("Invalid procedure for maximum rate limit selected for group" + group.name());
}
auto cc = Dune::MPIHelper::getCollectiveCommunication();
Parallel::Communication cc = comm_;
if (!ss.str().empty() && cc.rank() == 0)
deferred_logger.info(ss.str());
}
@ -1050,7 +1050,8 @@ actionOnBrokenConstraints(const Group& group,
<< " to " << Group::InjectionCMode2String(newControl);
this->groupState().injection_control(group.name(), controlPhase, newControl);
}
auto cc = Dune::MPIHelper::getCollectiveCommunication();
Parallel::Communication cc = comm_;
if (!ss.str().empty() && cc.rank() == 0)
deferred_logger.info(ss.str());
}
@ -1819,7 +1820,7 @@ updateWellPotentials(const int reportStepIdx,
}
logAndCheckForExceptionsAndThrow(deferred_logger, exc_type,
"computeWellPotentials() failed: " + exc_msg,
terminal_output_);
terminal_output_, comm_);
}

6
opm/simulators/wells/BlackoilWellModelGeneric.hpp
View File

@ -32,6 +32,7 @@
#include <unordered_set>
#include <vector>
#include <dune/common/version.hh>
#include <opm/output/data/GuideRateValue.hpp>
#include <opm/parser/eclipse/EclipseState/Schedule/Well/WellTestState.hpp>
#include <opm/parser/eclipse/EclipseState/Schedule/Group/GuideRate.hpp>
@ -68,7 +69,6 @@ class BlackoilWellModelGeneric
{
public:
// --------- Types ---------
using Comm = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
using GLiftOptWells = std::map<std::string,std::unique_ptr<GasLiftSingleWellGeneric>>;
using GLiftProdWells = std::map<std::string,const WellInterfaceGeneric*>;
using GLiftWellStateMap = std::map<std::string,std::unique_ptr<GasLiftWellState>>;
@ -77,7 +77,7 @@ public:
const SummaryState& summaryState,
const EclipseState& eclState,
const PhaseUsage& phase_usage,
const Comm& comm);
const Parallel::Communication& comm);
virtual ~BlackoilWellModelGeneric() = default;
@ -376,7 +376,7 @@ protected:
Schedule& schedule_;
const SummaryState& summaryState_;
const EclipseState& eclState_;
const Comm& comm_;
const Parallel::Communication& comm_;
PhaseUsage phase_usage_;
bool terminal_output_{false};

49
opm/simulators/wells/BlackoilWellModel_impl.hpp
View File

@ -59,8 +59,10 @@ namespace Opm {
cartDims[0] * cartDims[1] * cartDims[2]);
auto& parallel_wells = ebosSimulator.vanguard().parallelWells();
this->parallel_well_info_.assign(parallel_wells.begin(),
parallel_wells.end());
for (const auto& wellinfo : parallel_wells) {
this->parallel_well_info_.emplace_back(wellinfo, grid.comm());
}
}
this->alternative_well_rate_init_ =
@ -232,7 +234,7 @@ namespace Opm {
}
}
OPM_END_PARALLEL_TRY_CATCH_LOG(local_deferredLogger, "beginReportStep() failed: ",
terminal_output_);
terminal_output_, grid.comm());
// Store the current well state, to be able to recover in the case of failed iterations
this->commitWGState();
}
@ -246,7 +248,6 @@ namespace Opm {
{
updatePerforationIntensiveQuantities();
updateAverageFormationFactor();
DeferredLogger local_deferredLogger;
this->resetWGState();
@ -283,9 +284,10 @@ namespace Opm {
setRepRadiusPerfLength();
}
}
}
OPM_END_PARALLEL_TRY_CATCH_LOG(local_deferredLogger, "beginTimeStep() failed: ",
terminal_output_);
terminal_output_, ebosSimulator_.vanguard().grid().comm());
for (auto& well : well_container_) {
well->setVFPProperties(vfp_properties_.get());
@ -370,7 +372,7 @@ namespace Opm {
}
logAndCheckForExceptionsAndThrow(local_deferredLogger,
exc_type, "beginTimeStep() failed: " + exc_msg, terminal_output_);
exc_type, "beginTimeStep() failed: " + exc_msg, terminal_output_, comm);
}
@ -479,8 +481,9 @@ namespace Opm {
this->calculateProductivityIndexValues(local_deferredLogger);
this->commitWGState();
DeferredLogger global_deferredLogger = gatherDeferredLogger(local_deferredLogger);
const Opm::Parallel::Communication& comm = grid().comm();
DeferredLogger global_deferredLogger = gatherDeferredLogger(local_deferredLogger, comm);
if (terminal_output_) {
global_deferredLogger.logMessages();
}
@ -546,7 +549,7 @@ namespace Opm {
perf_pressure = fs.pressure(FluidSystem::gasPhaseIdx).value();
}
}
OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::initializeWellState() failed: ");
OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::initializeWellState() failed: ", ebosSimulator_.vanguard().grid().comm());
this->wellState().init(cellPressures, schedule(), wells_ecl_, local_parallel_well_info_, timeStepIdx,
&this->prevWellState(), well_perf_data_,
@ -691,7 +694,9 @@ namespace Opm {
}
// Collect log messages and print.
DeferredLogger global_deferredLogger = gatherDeferredLogger(local_deferredLogger);
const Opm::Parallel::Communication& comm = grid().comm();
DeferredLogger global_deferredLogger = gatherDeferredLogger(local_deferredLogger, comm);
if (terminal_output_) {
global_deferredLogger.logMessages();
}
@ -818,7 +823,7 @@ namespace Opm {
prepareTimeStep(local_deferredLogger);
}
OPM_END_PARALLEL_TRY_CATCH_LOG(local_deferredLogger, "assemble() failed (It=0): ",
terminal_output_);
terminal_output_, grid().comm());
}
updateWellControls(local_deferredLogger, /* check group controls */ true);
@ -831,7 +836,7 @@ namespace Opm {
assembleWellEq(dt, local_deferredLogger);
}
OPM_END_PARALLEL_TRY_CATCH_LOG(local_deferredLogger, "assemble() failed: ",
terminal_output_);
terminal_output_, grid().comm());
last_report_.converged = true;
last_report_.assemble_time_well += perfTimer.stop();
}
@ -1094,6 +1099,7 @@ namespace Opm {
BlackoilWellModel<TypeTag>::
recoverWellSolutionAndUpdateWellState(const BVector& x)
{
DeferredLogger local_deferredLogger;
OPM_BEGIN_PARALLEL_TRY_CATCH();
{
@ -1102,10 +1108,12 @@ namespace Opm {
well->recoverWellSolutionAndUpdateWellState(x, this->wellState(), local_deferredLogger);
}
}
}
OPM_END_PARALLEL_TRY_CATCH_LOG(local_deferredLogger,
"recoverWellSolutionAndUpdateWellState() failed: ",
terminal_output_);
terminal_output_, ebosSimulator_.vanguard().grid().comm());
}
@ -1141,12 +1149,14 @@ namespace Opm {
local_report += well->getWellConvergence(this->wellState(), B_avg, local_deferredLogger, iterationIdx > param_.strict_outer_iter_wells_ );
}
}
DeferredLogger global_deferredLogger = gatherDeferredLogger(local_deferredLogger);
const Opm::Parallel::Communication comm = grid().comm();
DeferredLogger global_deferredLogger = gatherDeferredLogger(local_deferredLogger, comm);
if (terminal_output_) {
global_deferredLogger.logMessages();
}
ConvergenceReport report = gatherConvergenceReport(local_report);
ConvergenceReport report = gatherConvergenceReport(local_report, comm);
// Log debug messages for NaN or too large residuals.
if (terminal_output_) {
@ -1293,8 +1303,9 @@ namespace Opm {
this->closed_this_step_.insert(wname);
}
}
DeferredLogger global_deferredLogger = gatherDeferredLogger(local_deferredLogger);
const Opm::Parallel::Communication comm = grid().comm();
DeferredLogger global_deferredLogger = gatherDeferredLogger(local_deferredLogger, comm);
if (terminal_output_) {
global_deferredLogger.logMessages();
}
@ -1475,7 +1486,7 @@ namespace Opm {
B += 1 / intQuants.solventInverseFormationVolumeFactor().value();
}
}
OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::updateAverageFormationFactor() failed: ")
OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::updateAverageFormationFactor() failed: ", grid.comm())
// compute global average
grid.comm().sum(B_avg.data(), B_avg.size());
@ -1565,7 +1576,7 @@ namespace Opm {
}
elemCtx.updatePrimaryIntensiveQuantities(/*timeIdx=*/0);
}
OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::updatePerforationIntensiveQuantities() failed: ");
OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::updatePerforationIntensiveQuantities() failed: ", ebosSimulator_.vanguard().grid().comm());
}

9
opm/simulators/wells/GasLiftGroupInfo.hpp
View File

@ -55,11 +55,10 @@ class GasLiftGroupInfo
using GroupRateMap =
std::map<std::string, GroupRates>;
using GroupIdxMap = std::map<std::string, int>;
using MPIComm = typename Dune::MPIHelper::MPICommunicator;
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<MPIComm>;
using Communication = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
#else
using Communication = Dune::CollectiveCommunication<MPIComm>;
using Communication = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
#endif
// TODO: same definition with WellInterface, and
@ -78,7 +77,7 @@ public:
const PhaseUsage& phase_usage,
DeferredLogger& deferred_logger,
WellState& well_state,
const Communication& comm);
const Parallel::Communication& comm);
std::vector<std::pair<std::string,double>>& getWellGroups(
const std::string& well_name);
@ -163,7 +162,7 @@ private:
const PhaseUsage &phase_usage_;
DeferredLogger &deferred_logger_;
WellState &well_state_;
const Communication &comm_;
const Parallel::Communication &comm_;
const GasLiftOpt& glo_;
GroupRateMap group_rate_map_;
Well2GroupMap well_group_map_;

2
opm/simulators/wells/GasLiftStage2.cpp
View File

@ -39,7 +39,7 @@ namespace Opm {
GasLiftStage2::GasLiftStage2(
const int report_step_idx,
const Communication& comm,
const Parallel::Communication& comm,
const Schedule& schedule,
const SummaryState& summary_state,
DeferredLogger &deferred_logger,

9
opm/simulators/wells/GasLiftStage2.hpp
View File

@ -55,18 +55,13 @@ class GasLiftStage2 {
using GradInfo = typename GasLiftSingleWellGeneric::GradInfo;
using GradMap = std::map<std::string, GradInfo>;
using MPIComm = typename Dune::MPIHelper::MPICommunicator;
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<MPIComm>;
#else
using Communication = Dune::CollectiveCommunication<MPIComm>;
#endif
static const int Water = BlackoilPhases::Aqua;
static const int Oil = BlackoilPhases::Liquid;
static const int Gas = BlackoilPhases::Vapour;
public:
GasLiftStage2(
const int report_step_idx,
const Communication& comm,
const Parallel::Communication& comm,
const Schedule& schedule,
const SummaryState& summary_state,
DeferredLogger& deferred_logger,
@ -136,7 +131,7 @@ private:
const SummaryState& summary_state_;
const Schedule& schedule_;
const GasLiftOpt& glo_;
const Communication& comm_;
const Parallel::Communication& comm_;
GradMap inc_grads_;
GradMap dec_grads_;
bool debug_;

4
opm/simulators/wells/ParallelWellInfo.cpp
View File

@ -357,7 +357,8 @@ ParallelWellInfo::ParallelWellInfo(const std::string& name,
commAboveBelow_(new CommunicateAboveBelow(*comm_))
{}
ParallelWellInfo::ParallelWellInfo(const std::pair<std::string,bool>& well_info,
ParallelWellInfo::ParallelWellInfo(const std::pair<std::string, bool>& well_info,
[[maybe_unused]] Communication allComm)
: name_(well_info.first), hasLocalCells_(well_info.second),
rankWithFirstPerf_(-1)
@ -374,6 +375,7 @@ ParallelWellInfo::ParallelWellInfo(const std::pair<std::string,bool>& well_info,
isOwner_ = (comm_->rank() == 0);
}
void ParallelWellInfo::communicateFirstPerforation(bool hasFirst)
{
int first = hasFirst;

3
opm/simulators/wells/ParallelWellInfo.hpp
View File

@ -255,7 +255,6 @@ public:
#else
using Communication = Dune::CollectiveCommunication<MPIComm>;
#endif
static constexpr int INVALID_ECL_INDEX = -1;
/// \brief Constructs object using MPI_COMM_SELF
@ -269,7 +268,7 @@ public:
/// \param allComm The communication object with all MPI ranks active in the simulation.
/// Default is the one with all ranks available.
ParallelWellInfo(const std::pair<std::string,bool>& well_info,
Communication allComm = Communication());
Communication allComm);
const Communication& communication() const
{

2
opm/simulators/wells/RateConverter.hpp
View File

@ -196,7 +196,7 @@ namespace Opm {
}
}
OPM_END_PARALLEL_TRY_CATCH("SurfaceToReservoirVoidage::defineState() failed: ");
OPM_END_PARALLEL_TRY_CATCH("SurfaceToReservoirVoidage::defineState() failed: ", simulator.vanguard().grid().comm());
for (const auto& reg : rmap_.activeRegions()) {
auto& ra = attr_.attributes(reg);

2
opm/simulators/wells/RegionAverageCalculator.hpp
View File

@ -175,7 +175,7 @@ namespace Opm {
}
}
}
OPM_END_PARALLEL_TRY_CATCH("AverageRegionalPressure::defineState(): ");
OPM_END_PARALLEL_TRY_CATCH("AverageRegionalPressure::defineState(): ", simulator.vanguard().grid().comm());
for (int reg = 1; reg <= numRegions ; ++ reg) {
auto& ra = attr_.attributes(reg);

2
opm/simulators/wells/StandardWellEval.hpp
View File

@ -85,7 +85,7 @@ protected:
static const bool gasEnabled = Indices::gasEnabled;
static const bool oilEnabled = Indices::oilEnabled;
static constexpr bool has_wfrac_variable = Indices::waterEnabled && Indices::oilEnabled;;
static constexpr bool has_wfrac_variable = Indices::waterEnabled && Indices::oilEnabled;
static constexpr bool has_gfrac_variable = Indices::gasEnabled && Indices::numPhases > 1;
static constexpr int WFrac = has_wfrac_variable ? 1 : -1000;
static constexpr int GFrac = has_gfrac_variable ? has_wfrac_variable + 1 : -1000;

5
opm/simulators/wells/WellInterface_impl.hpp
View File

@ -24,6 +24,8 @@
#include <opm/simulators/wells/GroupState.hpp>
#include <opm/simulators/wells/TargetCalculator.hpp>
#include <dune/common/version.hh>
namespace Opm
{
@ -201,8 +203,7 @@ namespace Opm
changed = this->checkConstraints(well_state, group_state, schedule, summaryState, deferred_logger);
}
auto cc = Dune::MPIHelper::getCollectiveCommunication();
Parallel::Communication cc = ebos_simulator.vanguard().grid().comm();
// checking whether control changed
if (changed) {
std::string to;

3
opm/simulators/wells/WellState.hpp
View File

@ -34,6 +34,9 @@
#include <opm/parser/eclipse/EclipseState/Schedule/Events.hpp>
#include <opm/parser/eclipse/EclipseState/Schedule/Well/Well.hpp>
#include <dune/common/version.hh>
#include <dune/common/parallel/mpihelper.hh>
#include <functional>
#include <map>
#include <optional>

5
tests/test_gatherconvergencereport.cpp
View File

@ -25,6 +25,7 @@
#include <boost/test/unit_test.hpp>
#include <dune/common/version.hh>
#include <opm/simulators/timestepping/gatherConvergenceReport.hpp>
#include <dune/common/parallel/mpihelper.hh>
@ -71,7 +72,7 @@ BOOST_AUTO_TEST_CASE(AllHaveFailure)
using CR = Opm::ConvergenceReport;
CR cr;
cr.setWellFailed({CR::WellFailure::Type::ControlBHP, CR::Severity::Normal, -1, name.str()});
CR global_cr = gatherConvergenceReport(cr);
CR global_cr = gatherConvergenceReport(cr, cc);
BOOST_CHECK(global_cr.wellFailures().size() == std::size_t(cc.size()));
BOOST_CHECK(global_cr.wellFailures()[cc.rank()] == cr.wellFailures()[0]);
// Extra output for debugging.
@ -92,7 +93,7 @@ BOOST_AUTO_TEST_CASE(EvenHaveFailure)
name << "WellRank" << cc.rank() << std::flush;
cr.setWellFailed({CR::WellFailure::Type::ControlBHP, CR::Severity::Normal, -1, name.str()});
}
CR global_cr = gatherConvergenceReport(cr);
CR global_cr = gatherConvergenceReport(cr, cc);
BOOST_CHECK(global_cr.wellFailures().size() == std::size_t((cc.size())+1) / 2);
if (cc.rank() % 2 == 0) {
BOOST_CHECK(global_cr.wellFailures()[cc.rank()/2] == cr.wellFailures()[0]);

13
tests/test_gatherdeferredlogger.cpp
View File

@ -25,6 +25,7 @@
#include <boost/test/unit_test.hpp>
#include <dune/common/version.hh>
#include <opm/simulators/utils/gatherDeferredLogger.hpp>
#include <dune/common/parallel/mpihelper.hh>
@ -80,14 +81,14 @@ void initLogger(std::ostringstream& log_stream) {
BOOST_AUTO_TEST_CASE(NoMessages)
{
auto cc = Dune::MPIHelper::getCollectiveCommunication();
const Opm::Parallel::Communication& cc = Dune::MPIHelper::getCollectiveCommunication();
std::ostringstream log_stream;
initLogger(log_stream);
Opm::DeferredLogger local_deferredlogger;
Opm::DeferredLogger global_deferredlogger = gatherDeferredLogger(local_deferredlogger);
Opm::DeferredLogger global_deferredlogger = gatherDeferredLogger(local_deferredlogger, cc);
if (cc.rank() == 0) {
@ -103,7 +104,7 @@ BOOST_AUTO_TEST_CASE(NoMessages)
BOOST_AUTO_TEST_CASE(VariableNumberOfMessages)
{
auto cc = Dune::MPIHelper::getCollectiveCommunication();
const Opm::Parallel::Communication& cc = Dune::MPIHelper::getCollectiveCommunication();
std::ostringstream log_stream;
initLogger(log_stream);
@ -119,7 +120,7 @@ BOOST_AUTO_TEST_CASE(VariableNumberOfMessages)
local_deferredlogger.bug("tagme", "bug from rank " + std::to_string(cc.rank()));
}
Opm::DeferredLogger global_deferredlogger = gatherDeferredLogger(local_deferredlogger);
Opm::DeferredLogger global_deferredlogger = gatherDeferredLogger(local_deferredlogger, cc);
if (cc.rank() == 0) {
@ -141,7 +142,7 @@ BOOST_AUTO_TEST_CASE(VariableNumberOfMessages)
BOOST_AUTO_TEST_CASE(AllHaveOneMessage)
{
auto cc = Dune::MPIHelper::getCollectiveCommunication();
const Opm::Parallel::Communication& cc = Dune::MPIHelper::getCollectiveCommunication();
std::ostringstream log_stream;
initLogger(log_stream);
@ -149,7 +150,7 @@ BOOST_AUTO_TEST_CASE(AllHaveOneMessage)
Opm::DeferredLogger local_deferredlogger;
local_deferredlogger.info("info from rank " + std::to_string(cc.rank()));
Opm::DeferredLogger global_deferredlogger = gatherDeferredLogger(local_deferredlogger);
Opm::DeferredLogger global_deferredlogger = gatherDeferredLogger(local_deferredlogger, cc);
if (cc.rank() == 0) {

34
tests/test_parallelwellinfo.cpp
View File

@ -18,7 +18,10 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include<config.h>
#include<opm/simulators/wells/ParallelWellInfo.hpp>
#include <dune/common/version.hh>
#include<vector>
#include<string>
#include<tuple>
@ -105,7 +108,7 @@ BOOST_AUTO_TEST_CASE(ParallelWellComparison)
{
int argc = 0;
char** argv = nullptr;
const auto& helper = Dune::MPIHelper::instance(argc, argv);
const auto& helper = Dune::MPIHelper::instance(argc, argv);
std::vector<std::pair<std::string,bool>> pairs;
if (helper.rank() == 0)
pairs = {{"Test1", true},{"Test2", true}, {"Test1", false} };
@ -113,7 +116,12 @@ BOOST_AUTO_TEST_CASE(ParallelWellComparison)
pairs = {{"Test1", false},{"Test2", true}, {"Test1", true} };
std::vector<Opm::ParallelWellInfo> well_info;
well_info.assign(pairs.begin(), pairs.end());
for (const auto& wellinfo : pairs) {
well_info.emplace_back(wellinfo, Opm::ParallelWellInfo::Communication());
}
//well_info.assign(pairs.begin(), pairs.end());
BOOST_CHECK_EQUAL_COLLECTIONS(pairs.begin(), pairs.end(),
well_info.begin(), well_info.end());
@ -227,13 +235,7 @@ BOOST_AUTO_TEST_CASE(CommunicateAboveBelowSelf1)
}
}
using MPIComm = typename Dune::MPIHelper::MPICommunicator;
#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
using Communication = Dune::Communication<MPIComm>;
#else
using Communication = Dune::CollectiveCommunication<MPIComm>;
#endif
std::vector<int> createGlobalEclIndex(const Communication& comm)
std::vector<int> createGlobalEclIndex(const Opm::ParallelWellInfo::Communication& comm)
{
std::vector<int> globalEclIndex = {0, 1, 2, 3, 7 , 8, 10, 11};
auto oldSize = globalEclIndex.size();
@ -254,7 +256,7 @@ std::vector<int> createGlobalEclIndex(const Communication& comm)
template<class C>
std::vector<double> populateCommAbove(C& commAboveBelow,
const Communication& comm,
const Opm::ParallelWellInfo::Communication& comm,
const std::vector<int>& globalEclIndex,
const std::vector<double> globalCurrent,
int num_component = 1,
@ -285,7 +287,7 @@ std::vector<double> populateCommAbove(C& commAboveBelow,
BOOST_AUTO_TEST_CASE(CommunicateAboveBelowParallel)
{
auto comm = Communication(Dune::MPIHelper::getCommunicator());
auto comm = Opm::ParallelWellInfo::Communication(Dune::MPIHelper::getCommunicator());
Opm::CommunicateAboveBelow commAboveBelow{ comm };
for(std::size_t count=0; count < 2; ++count)
@ -363,7 +365,7 @@ BOOST_AUTO_TEST_CASE(PartialSumself)
commAboveBelow.endReset();
initRandomNumbers(std::begin(current), std::end(current),
Communication(comm));
Opm::ParallelWellInfo::Communication(comm));
auto stdCopy = current;
std::partial_sum(std::begin(stdCopy), std::end(stdCopy), std::begin(stdCopy));
@ -377,13 +379,13 @@ BOOST_AUTO_TEST_CASE(PartialSumself)
BOOST_AUTO_TEST_CASE(PartialSumParallel)
{
auto comm = Communication(Dune::MPIHelper::getCommunicator());
auto comm = Opm::ParallelWellInfo::Communication(Dune::MPIHelper::getCommunicator());
Opm::CommunicateAboveBelow commAboveBelow{ comm };
auto globalEclIndex = createGlobalEclIndex(comm);
std::vector<double> globalCurrent(globalEclIndex.size());
initRandomNumbers(std::begin(globalCurrent), std::end(globalCurrent),
Communication(comm));
Opm::ParallelWellInfo::Communication(comm));
auto localCurrent = populateCommAbove(commAboveBelow, comm,
globalEclIndex, globalCurrent);
@ -405,7 +407,7 @@ BOOST_AUTO_TEST_CASE(PartialSumParallel)
void testGlobalPerfFactoryParallel(int num_component, bool local_consecutive = false)
{
auto comm = Communication(Dune::MPIHelper::getCommunicator());
auto comm = Opm::ParallelWellInfo::Communication(Dune::MPIHelper::getCommunicator());
Opm::ParallelWellInfo wellInfo{ {"Test", true }, comm };
auto globalEclIndex = createGlobalEclIndex(comm);
@ -471,7 +473,7 @@ BOOST_AUTO_TEST_CASE(GlobalPerfFactoryParallel1)
BOOST_AUTO_TEST_CASE(EmptyWell) {
auto comm = Communication(Dune::MPIHelper::getCommunicator());
auto comm = Opm::ParallelWellInfo::Communication(Dune::MPIHelper::getCommunicator());
Opm::ParallelWellInfo pw({"WELL1", true}, comm);
pw.communicateFirstPerforation(false);
double local_p = 1;

2
tests/test_wellmodel.cpp
View File

@ -102,7 +102,7 @@ struct GlobalFixture {
Dune::MPIHelper::instance(argcDummy, argvDummy);
#endif
Opm::FlowMainEbos<Opm::Properties::TTag::EclFlowProblem>::setupParameters_(argcDummy, argvDummy);
Opm::FlowMainEbos<Opm::Properties::TTag::EclFlowProblem>::setupParameters_(argcDummy, argvDummy, Dune::MPIHelper::getCollectiveCommunication());
}
};