From 8e9cb4b249cd94a3d8ef211716fd3abe7fd306c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A5rd=20Skaflestad?= Date: Mon, 4 Mar 2024 09:08:58 +0100 Subject: [PATCH 1/2] Defer SummaryConfig Initialisation In the current approach, the full list of summary vectors, especially those that are defined at the region level, is not known until we've processed all UDQ and/or ACTIONX definitions. As a quick solution to this, switch to using the 'SummaryConfig' object that's defined in the EclipseIO container instead of the object that gets constructed when reading the input files. It is likely that we will have to rethink and refactor this construction process later. --- opm/simulators/flow/DamarisWriter.hpp | 56 +++++++++++++++---- opm/simulators/flow/EclGenericWriter.hpp | 1 - opm/simulators/flow/EclGenericWriter_impl.hpp | 5 +- opm/simulators/flow/EclWriter.hpp | 32 +++++++++-- .../flow/GenericOutputBlackoilModule.cpp | 2 +- .../flow/GenericOutputBlackoilModule.hpp | 4 +- opm/simulators/flow/OutputBlackoilModule.hpp | 7 +-- 7 files changed, 83 insertions(+), 24 deletions(-) diff --git a/opm/simulators/flow/DamarisWriter.hpp b/opm/simulators/flow/DamarisWriter.hpp index d46bdae56..263452618 100644 --- a/opm/simulators/flow/DamarisWriter.hpp +++ b/opm/simulators/flow/DamarisWriter.hpp @@ -36,6 +36,9 @@ #include +#include +#include + #include #include #include @@ -149,25 +152,56 @@ public: simulator.vanguard().eclState(), simulator.vanguard().summaryConfig(), simulator.vanguard().grid(), - simulator.vanguard().grid().comm().rank() == 0 ? &simulator.vanguard().equilGrid() : nullptr, + ((simulator.vanguard().grid().comm().rank() == 0) + ? &simulator.vanguard().equilGrid() + : nullptr), simulator.vanguard().gridView(), simulator.vanguard().cartesianIndexMapper(), - simulator.vanguard().grid().comm().rank() == 0 ? &simulator.vanguard().equilCartesianIndexMapper() : nullptr, + ((simulator.vanguard().grid().comm().rank() == 0) + ? &simulator.vanguard().equilCartesianIndexMapper() + : nullptr), false, false) , simulator_(simulator) { this->damarisUpdate_ = true ; - rank_ = simulator_.vanguard().grid().comm().rank() ; - nranks_ = simulator_.vanguard().grid().comm().size(); - - const auto& gridView = simulator_.gridView(); - const auto& interior_elements = elements(gridView, Dune::Partitions::interior); - // Get the size of the unique vector elements (excludes the shared 'ghost' elements) - numElements_ = std::distance(interior_elements.begin(), interior_elements.end()); + this->rank_ = this->simulator_.vanguard().grid().comm().rank() ; + this->nranks_ = this->simulator_.vanguard().grid().comm().size(); - this->elements_rank_offsets_.resize(nranks_) ; - this->damarisOutputModule_ = std::make_unique>(simulator, this->collectOnIORank_); + this->elements_rank_offsets_.resize(this->nranks_); + + // Get the size of the unique vector elements (excludes the shared 'ghost' elements) + // + // Might possibly use + // + // detail::countLocalInteriorCellsGridView(this->simulator_.gridView()) + // + // from countGlobalCells.hpp instead of calling std::distance() directly. + { + const auto& gridView = this->simulator_.gridView(); + const auto& interior_elements = elements(gridView, Dune::Partitions::interior); + + this->numElements_ = std::distance(interior_elements.begin(), interior_elements.end()); + } + + if (this->nranks_ > 1) { + auto smryCfg = (this->rank_ == 0) + ? this->eclIO_->finalSummaryConfig() + : SummaryConfig{}; + + auto serialiser = Parallel::MpiSerializer { + this->simulator_.vanguard().grid().comm() + }; + + serialiser.broadcast(0, smryCfg); + + this->damarisOutputModule_ = std::make_unique> + (simulator, smryCfg, this->collectOnIORank_); + } + else { + this->damarisOutputModule_ = std::make_unique> + (simulator, this->eclIO_->finalSummaryConfig(), this->collectOnIORank_); + } } /*! diff --git a/opm/simulators/flow/EclGenericWriter.hpp b/opm/simulators/flow/EclGenericWriter.hpp index 01e4d2262..b2f43dfd6 100644 --- a/opm/simulators/flow/EclGenericWriter.hpp +++ b/opm/simulators/flow/EclGenericWriter.hpp @@ -154,7 +154,6 @@ protected: const GridView& gridView_; const Schedule& schedule_; const EclipseState& eclState_; - const SummaryConfig& summaryConfig_; std::unique_ptr eclIO_; std::unique_ptr taskletRunner_; Scalar restartTimeStepSize_; diff --git a/opm/simulators/flow/EclGenericWriter_impl.hpp b/opm/simulators/flow/EclGenericWriter_impl.hpp index 1a2d94e5b..48c6fe3e9 100644 --- a/opm/simulators/flow/EclGenericWriter_impl.hpp +++ b/opm/simulators/flow/EclGenericWriter_impl.hpp @@ -30,12 +30,14 @@ #include #include + #include #include #include #include #include #include + #include #include @@ -207,7 +209,6 @@ EclGenericWriter(const Schedule& schedule, , gridView_ (gridView) , schedule_ (schedule) , eclState_ (eclState) - , summaryConfig_ (summaryConfig) , cartMapper_ (cartMapper) , equilCartMapper_(equilCartMapper) , equilGrid_ (equilGrid) @@ -216,7 +217,7 @@ EclGenericWriter(const Schedule& schedule, this->eclIO_ = std::make_unique (this->eclState_, UgGridHelpers::createEclipseGrid(*equilGrid, eclState_.getInputGrid()), - this->schedule_, this->summaryConfig_, "", enableEsmry); + this->schedule_, summaryConfig, "", enableEsmry); } // create output thread if enabled and rank is I/O rank diff --git a/opm/simulators/flow/EclWriter.hpp b/opm/simulators/flow/EclWriter.hpp index 03cd7e598..88a5ab8cf 100644 --- a/opm/simulators/flow/EclWriter.hpp +++ b/opm/simulators/flow/EclWriter.hpp @@ -36,6 +36,11 @@ #include +#if HAVE_MPI +#include +#include +#endif + #include #include #include @@ -151,10 +156,29 @@ public: EWOMS_GET_PARAM(TypeTag, bool, EnableEsmry)) , simulator_(simulator) { - this->outputModule_ = std::make_unique> - (simulator, this->collectOnIORank_); - - rank_ = simulator_.vanguard().grid().comm().rank() ; +#if HAVE_MPI + if (this->simulator_.vanguard().grid().comm().size() > 1) { + auto smryCfg = (this->simulator_.vanguard().grid().comm().rank() == 0) + ? this->eclIO_->finalSummaryConfig() + : SummaryConfig{}; + + auto serialiser = Parallel::MpiSerializer { + this->simulator_.vanguard().grid().comm() + }; + + serialiser.broadcast(0, smryCfg); + + this->outputModule_ = std::make_unique> + (simulator, smryCfg, this->collectOnIORank_); + } + else +#endif + { + this->outputModule_ = std::make_unique> + (simulator, this->eclIO_->finalSummaryConfig(), this->collectOnIORank_); + } + + this->rank_ = this->simulator_.vanguard().grid().comm().rank(); } ~EclWriter() diff --git a/opm/simulators/flow/GenericOutputBlackoilModule.cpp b/opm/simulators/flow/GenericOutputBlackoilModule.cpp index 814db6aa1..a99a17727 100644 --- a/opm/simulators/flow/GenericOutputBlackoilModule.cpp +++ b/opm/simulators/flow/GenericOutputBlackoilModule.cpp @@ -184,8 +184,8 @@ GenericOutputBlackoilModule(const EclipseState& eclState, bool enableMICP) : eclState_(eclState) , schedule_(schedule) - , summaryConfig_(summaryConfig) , summaryState_(summaryState) + , summaryConfig_(summaryConfig) , interRegionFlows_(numCells(eclState), defineInterRegionFlowArrays(eclState, summaryConfig), declaredMaxRegionID(eclState.runspec())) diff --git a/opm/simulators/flow/GenericOutputBlackoilModule.hpp b/opm/simulators/flow/GenericOutputBlackoilModule.hpp index 984f6e1c7..4eb621694 100644 --- a/opm/simulators/flow/GenericOutputBlackoilModule.hpp +++ b/opm/simulators/flow/GenericOutputBlackoilModule.hpp @@ -24,6 +24,7 @@ #define OPM_GENERIC_OUTPUT_BLACK_OIL_MODULE_HPP #include +#include #include #include @@ -376,9 +377,10 @@ protected: const EclipseState& eclState_; const Schedule& schedule_; - const SummaryConfig& summaryConfig_; const SummaryState& summaryState_; + SummaryConfig summaryConfig_; + InterRegFlowMap interRegionFlows_; LogOutputHelper logOutput_; diff --git a/opm/simulators/flow/OutputBlackoilModule.hpp b/opm/simulators/flow/OutputBlackoilModule.hpp index 1e64bb2f8..f25c783fc 100644 --- a/opm/simulators/flow/OutputBlackoilModule.hpp +++ b/opm/simulators/flow/OutputBlackoilModule.hpp @@ -143,10 +143,11 @@ class OutputBlackOilModule : public GenericOutputBlackoilModule OutputBlackOilModule(const Simulator& simulator, + const SummaryConfig& smryCfg, const CollectDataToIORankType& collectToIORank) : BaseType(simulator.vanguard().eclState(), simulator.vanguard().schedule(), - simulator.vanguard().summaryConfig(), + smryCfg, simulator.vanguard().summaryState(), moduleVersionName(), getPropValue(), @@ -185,9 +186,7 @@ public: OPM_THROW_NOLOG(std::runtime_error, msg); } - if (const auto& smryCfg = simulator.vanguard().summaryConfig(); - smryCfg.match("[FB]PP[OGW]") || smryCfg.match("RPP[OGW]*")) - { + if (smryCfg.match("[FB]PP[OGW]") || smryCfg.match("RPP[OGW]*")) { auto rset = this->eclState_.fieldProps().fip_regions(); rset.push_back("PVTNUM"); From 785b96f6f3fedee6a45a991b4eacded2e60f4162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A5rd=20Skaflestad?= Date: Fri, 22 Mar 2024 14:59:10 +0100 Subject: [PATCH 2/2] Distribute SummaryConfig Objects With eclBroadcast Suggested by [at]akva2. While here, also switch to using type deduction instead of explicit template arguments for the eclBroadcast overloads. --- opm/simulators/flow/DamarisWriter.hpp | 10 ++-------- opm/simulators/flow/EclWriter.hpp | 12 ++---------- opm/simulators/utils/ParallelSerialization.cpp | 7 ++++--- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/opm/simulators/flow/DamarisWriter.hpp b/opm/simulators/flow/DamarisWriter.hpp index 263452618..775fe4c6d 100644 --- a/opm/simulators/flow/DamarisWriter.hpp +++ b/opm/simulators/flow/DamarisWriter.hpp @@ -36,9 +36,6 @@ #include -#include -#include - #include #include #include @@ -47,6 +44,7 @@ #include #include #include +#include #include @@ -189,11 +187,7 @@ public: ? this->eclIO_->finalSummaryConfig() : SummaryConfig{}; - auto serialiser = Parallel::MpiSerializer { - this->simulator_.vanguard().grid().comm() - }; - - serialiser.broadcast(0, smryCfg); + eclBroadcast(this->simulator_.vanguard().grid().comm(), smryCfg); this->damarisOutputModule_ = std::make_unique> (simulator, smryCfg, this->collectOnIORank_); diff --git a/opm/simulators/flow/EclWriter.hpp b/opm/simulators/flow/EclWriter.hpp index 88a5ab8cf..7f2104078 100644 --- a/opm/simulators/flow/EclWriter.hpp +++ b/opm/simulators/flow/EclWriter.hpp @@ -36,11 +36,6 @@ #include -#if HAVE_MPI -#include -#include -#endif - #include #include #include @@ -49,6 +44,7 @@ #include #include #include +#include #include #include @@ -162,11 +158,7 @@ public: ? this->eclIO_->finalSummaryConfig() : SummaryConfig{}; - auto serialiser = Parallel::MpiSerializer { - this->simulator_.vanguard().grid().comm() - }; - - serialiser.broadcast(0, smryCfg); + eclBroadcast(this->simulator_.vanguard().grid().comm(), smryCfg); this->outputModule_ = std::make_unique> (simulator, smryCfg, this->collectOnIORank_); diff --git a/opm/simulators/utils/ParallelSerialization.cpp b/opm/simulators/utils/ParallelSerialization.cpp index 34873f658..fa50ea454 100644 --- a/opm/simulators/utils/ParallelSerialization.cpp +++ b/opm/simulators/utils/ParallelSerialization.cpp @@ -82,11 +82,12 @@ void eclStateBroadcast(Parallel::Communication comm, EclipseState& eclState, Sch template void eclBroadcast(Parallel::Communication comm, T& data) { - Opm::Parallel::MpiSerializer ser(comm); + ::Opm::Parallel::MpiSerializer ser(comm); ser.broadcast(data); } -template void eclBroadcast(Parallel::Communication, TransMult&); -template void eclBroadcast(Parallel::Communication, Schedule&); +template void eclBroadcast(Parallel::Communication, TransMult&); +template void eclBroadcast(Parallel::Communication, Schedule&); +template void eclBroadcast(Parallel::Communication, SummaryConfig&); }