From 1bb7b0d30f4e43fc8ced4a1e1a6bb9b0c4cb8d76 Mon Sep 17 00:00:00 2001 From: Joakim Hove Date: Mon, 30 Nov 2020 13:31:04 +0100 Subject: [PATCH 1/3] Use unique_ptr for outputmodule member in EclWriter --- ebos/eclwriter.hh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ebos/eclwriter.hh b/ebos/eclwriter.hh index b7550ed9c..5444e7fb3 100644 --- a/ebos/eclwriter.hh +++ b/ebos/eclwriter.hh @@ -191,8 +191,8 @@ public: EclWriter(Simulator& simulator) : simulator_(simulator) , collectToIORank_(simulator_.vanguard()) - , eclOutputModule_(simulator, collectToIORank_) { + this->eclOutputModule_ = std::make_unique>(simulator, this->collectToIORank_); if (collectToIORank_.isIORank()) { eclIO_.reset(new Opm::EclipseIO(simulator_.vanguard().eclState(), Opm::UgGridHelpers::createEclipseGrid(globalGrid(), simulator_.vanguard().eclState().getInputGrid()), @@ -279,19 +279,19 @@ public: this->prepareLocalCellData(isSubStep, reportStepNum); if (collectToIORank_.isParallel()) - collectToIORank_.collect({}, eclOutputModule_.getBlockData(), + collectToIORank_.collect({}, eclOutputModule_->getBlockData(), localWellData, localGroupAndNetworkData); std::map miscSummaryData; std::map> regionData; - auto inplace = eclOutputModule_.outputFipLog(miscSummaryData, regionData, isSubStep); + auto inplace = eclOutputModule_->outputFipLog(miscSummaryData, regionData, isSubStep); bool forceDisableProdOutput = false; bool forceDisableInjOutput = false; bool forceDisableCumOutput = false; - eclOutputModule_.outputProdLog(reportStepNum, isSubStep, forceDisableProdOutput); - eclOutputModule_.outputInjLog(reportStepNum, isSubStep, forceDisableInjOutput); - eclOutputModule_.outputCumLog(reportStepNum, isSubStep, forceDisableCumOutput); + eclOutputModule_->outputProdLog(reportStepNum, isSubStep, forceDisableProdOutput); + eclOutputModule_->outputInjLog(reportStepNum, isSubStep, forceDisableInjOutput); + eclOutputModule_->outputCumLog(reportStepNum, isSubStep, forceDisableCumOutput); std::vector buffer; if (this->collectToIORank_.isIORank()) { @@ -316,7 +316,7 @@ public: const auto& blockData = this->collectToIORank_.isParallel() ? this->collectToIORank_.globalBlockData() - : this->eclOutputModule_.getBlockData(); + : this->eclOutputModule_->getBlockData(); summary.eval(summaryState(), reportStepNum, @@ -324,7 +324,7 @@ public: wellData, groupAndNetworkData, miscSummaryData, - eclOutputModule_.initialInplace(), + eclOutputModule_->initialInplace(), inplace, wbp_calculators, regionData, @@ -359,7 +359,7 @@ public: const int reportStepNum = simulator_.episodeIndex() + 1; this->prepareLocalCellData(isSubStep, reportStepNum); - this->eclOutputModule_.outputErrorLog(); + this->eclOutputModule_->outputErrorLog(); // output using eclWriter if enabled auto localWellData = simulator_.problem().wellModel().wellData(); @@ -368,14 +368,14 @@ public: Opm::data::Solution localCellData = {}; if (! isSubStep) { - this->eclOutputModule_.assignToSolution(localCellData); + this->eclOutputModule_->assignToSolution(localCellData); // add cell data to perforations for Rft output - this->eclOutputModule_.addRftDataToWells(localWellData, reportStepNum); + this->eclOutputModule_->addRftDataToWells(localWellData, reportStepNum); } if (this->collectToIORank_.isParallel()) { - collectToIORank_.collect(localCellData, eclOutputModule_.getBlockData(), + collectToIORank_.collect(localCellData, eclOutputModule_->getBlockData(), localWellData, localGroupAndNetworkData); } @@ -420,7 +420,7 @@ public: const auto& gridView = simulator_.vanguard().gridView(); unsigned numElements = gridView.size(/*codim=*/0); - eclOutputModule_.allocBuffers(numElements, restartStepIdx, /*isSubStep=*/false, /*log=*/false, /*isRestart*/ true); + eclOutputModule_->allocBuffers(numElements, restartStepIdx, /*isSubStep=*/false, /*log=*/false, /*isRestart*/ true); { Opm::SummaryState& summaryState = simulator_.vanguard().summaryState(); @@ -429,7 +429,7 @@ public: gridView.grid().comm()); for (unsigned elemIdx = 0; elemIdx < numElements; ++elemIdx) { unsigned globalIdx = collectToIORank_.localIdxToGlobalIdx(elemIdx); - eclOutputModule_.setRestart(restartValues.solution, elemIdx, globalIdx); + eclOutputModule_->setRestart(restartValues.solution, elemIdx, globalIdx); } if (inputThpres.active()) { @@ -452,7 +452,7 @@ public: {} const EclOutputBlackOilModule& eclOutputModule() const - { return eclOutputModule_; } + { return *eclOutputModule_; } Scalar restartTimeStepSize() const { return restartTimeStepSize_; } @@ -717,7 +717,7 @@ private: const int numElements = gridView.size(/*codim=*/0); const bool log = collectToIORank_.isIORank(); - eclOutputModule_.allocBuffers(numElements, reportStepNum, + eclOutputModule_->allocBuffers(numElements, reportStepNum, isSubStep, log, /*isRestart*/ false); ElementContext elemCtx(simulator_); @@ -730,7 +730,7 @@ private: elemCtx.updatePrimaryStencil(elem); elemCtx.updatePrimaryIntensiveQuantities(/*timeIdx=*/0); - eclOutputModule_.processElement(elemCtx); + eclOutputModule_->processElement(elemCtx); } } @@ -784,7 +784,7 @@ private: Simulator& simulator_; CollectDataToIORankType collectToIORank_; - EclOutputBlackOilModule eclOutputModule_; + std::unique_ptr> eclOutputModule_; std::unique_ptr eclIO_; std::unique_ptr taskletRunner_; Scalar restartTimeStepSize_; From 1d86e2dc97629e925cb1e566798d6d42abd8cc2b Mon Sep 17 00:00:00 2001 From: Joakim Hove Date: Tue, 1 Dec 2020 07:52:58 +0100 Subject: [PATCH 2/3] Pass wbp index list to output module --- ebos/ecloutputblackoilmodule.hh | 2 +- ebos/eclwriter.hh | 21 ++++++++++++++++++--- tests/test_ecl_output.cc | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/ebos/ecloutputblackoilmodule.hh b/ebos/ecloutputblackoilmodule.hh index 72fcbb666..e448d6e2a 100644 --- a/ebos/ecloutputblackoilmodule.hh +++ b/ebos/ecloutputblackoilmodule.hh @@ -195,7 +195,7 @@ class EclOutputBlackOilModule public: template - EclOutputBlackOilModule(const Simulator& simulator, const CollectDataToIORankType& collectToIORank) + EclOutputBlackOilModule(const Simulator& simulator, const std::vector& wbp_index_list, const CollectDataToIORankType& collectToIORank) : simulator_(simulator) { const Opm::SummaryConfig summaryConfig = simulator_.vanguard().summaryConfig(); diff --git a/ebos/eclwriter.hh b/ebos/eclwriter.hh index 5444e7fb3..6291524be 100644 --- a/ebos/eclwriter.hh +++ b/ebos/eclwriter.hh @@ -192,14 +192,27 @@ public: : simulator_(simulator) , collectToIORank_(simulator_.vanguard()) { - this->eclOutputModule_ = std::make_unique>(simulator, this->collectToIORank_); + std::vector wbp_index_list; if (collectToIORank_.isIORank()) { + const auto& schedule = simulator_.vanguard().schedule(); eclIO_.reset(new Opm::EclipseIO(simulator_.vanguard().eclState(), Opm::UgGridHelpers::createEclipseGrid(globalGrid(), simulator_.vanguard().eclState().getInputGrid()), - simulator_.vanguard().schedule(), + schedule, simulator_.vanguard().summaryConfig())); - } + const auto& wbp_calculators = eclIO_->summary().wbp_calculators( schedule.size() - 1 ); + wbp_index_list = wbp_calculators.index_list(); + } + if (collectToIORank_.isParallel()) { +#ifdef HAVE_MPI + unsigned long size = wbp_index_list.size(); + MPI_Bcast(&size, 1, MPI_UNSIGNED_LONG, collectToIORank_.ioRank, MPI_COMM_WORLD); + if (!collectToIORank_.isIORank()) + wbp_index_list.resize( size ); + + MPI_Bcast(wbp_index_list.data(), size * sizeof(std::size_t), MPI_CHAR, collectToIORank_.ioRank, MPI_COMM_WORLD); +#endif + } // create output thread if enabled and rank is I/O rank // async output is enabled by default if pthread are enabled bool enableAsyncOutput = EWOMS_GET_PARAM(TypeTag, bool, EnableAsyncEclOutput); @@ -207,6 +220,8 @@ public: if (enableAsyncOutput && collectToIORank_.isIORank()) numWorkerThreads = 1; taskletRunner_.reset(new TaskletRunner(numWorkerThreads)); + + this->eclOutputModule_ = std::make_unique>(simulator, wbp_index_list, this->collectToIORank_); } ~EclWriter() diff --git a/tests/test_ecl_output.cc b/tests/test_ecl_output.cc index 0854d7db8..e4984fee7 100644 --- a/tests/test_ecl_output.cc +++ b/tests/test_ecl_output.cc @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(Summary) using Vanguard = Opm::GetPropType; typedef Opm::CollectDataToIORank< Vanguard > CollectDataToIORankType; CollectDataToIORankType collectToIORank(simulator->vanguard()); - Opm::EclOutputBlackOilModule eclOutputModule(*simulator, collectToIORank); + Opm::EclOutputBlackOilModule eclOutputModule(*simulator, {}, collectToIORank); typedef Opm::EclWriter EclWriterType; // create the actual ECL writer From 72905f3e3b5a6150f16a3a36e858a5c79625e262 Mon Sep 17 00:00:00 2001 From: Joakim Hove Date: Tue, 1 Dec 2020 09:09:06 +0100 Subject: [PATCH 3/3] Gather WBP pressure data before summary eval --- ebos/collecttoiorank.hh | 69 ++++++++++++++++++++++++++++++++- ebos/ecloutputblackoilmodule.hh | 12 ++++++ ebos/eclwriter.hh | 32 ++++++++++----- 3 files changed, 101 insertions(+), 12 deletions(-) diff --git a/ebos/collecttoiorank.hh b/ebos/collecttoiorank.hh index 0bb1395a9..3efcd8822 100644 --- a/ebos/collecttoiorank.hh +++ b/ebos/collecttoiorank.hh @@ -655,14 +655,70 @@ public: }; + class PackUnPackWBPData : public P2PCommunicatorType::DataHandleInterface + { + const std::map& localWBPData_; + std::map& globalWBPValues_; + + public: + PackUnPackWBPData(const std::map& localWBPData, + std::map& globalWBPValues, + bool isIORank) + : localWBPData_(localWBPData) + , globalWBPValues_(globalWBPValues) + { + if (isIORank) { + MessageBufferType buffer; + pack(0, buffer); + + // pass a dummyLink to satisfy virtual class + int dummyLink = -1; + unpack(dummyLink, buffer); + } + } + + // pack all data associated with link + void pack(int link, MessageBufferType& buffer) + { + // we should only get one link + if (link != 0) + throw std::logic_error("link in method pack is not 0 as expected"); + + // write all block data + unsigned int size = localWBPData_.size(); + buffer.write(size); + for (const auto& [global_index, wbp_value] : localWBPData_) { + buffer.write(global_index); + buffer.write(wbp_value); + } + } + + // unpack all data associated with link + void unpack(int /*link*/, MessageBufferType& buffer) + { + // read all block data + unsigned int size = 0; + buffer.read(size); + for (size_t i = 0; i < size; ++i) { + std::size_t idx; + double data; + buffer.read(idx); + buffer.read(data); + globalWBPValues_[idx] = data; + } + } + + }; // gather solution to rank 0 for EclipseWriter void collect(const Opm::data::Solution& localCellData, const std::map, double>& localBlockData, + const std::map& localWBPData, const Opm::data::Wells& localWellData, const Opm::data::GroupAndNetworkValues& localGroupAndNetworkData) { globalCellData_ = {}; globalBlockData_.clear(); + globalWBPData_.clear(); globalWellData_.clear(); globalGroupAndNetworkData_.clear(); @@ -703,12 +759,17 @@ public: this->isIORank() }; + PackUnPackWBPData packUnpackWBPData { + localWBPData, + this->globalWBPData_, + this->isIORank() + }; + toIORankComm_.exchange(packUnpackCellData); toIORankComm_.exchange(packUnpackWellData); toIORankComm_.exchange(packUnpackGroupAndNetworkData); toIORankComm_.exchange(packUnpackBlockData); - - + toIORankComm_.exchange(packUnpackWBPData); #ifndef NDEBUG // mkae sure every process is on the same page @@ -716,6 +777,9 @@ public: #endif } + const std::map& globalWBPData() const + { return this->globalWBPData_; } + const std::map, double>& globalBlockData() const { return globalBlockData_; } @@ -772,6 +836,7 @@ protected: std::vector globalRanks_; Opm::data::Solution globalCellData_; std::map, double> globalBlockData_; + std::map globalWBPData_; Opm::data::Wells globalWellData_; Opm::data::GroupAndNetworkValues globalGroupAndNetworkData_; std::vector localIdxToGlobalIdx_; diff --git a/ebos/ecloutputblackoilmodule.hh b/ebos/ecloutputblackoilmodule.hh index e448d6e2a..895427b90 100644 --- a/ebos/ecloutputblackoilmodule.hh +++ b/ebos/ecloutputblackoilmodule.hh @@ -225,6 +225,11 @@ public: } } + for (const auto& global_index : wbp_index_list) { + if (collectToIORank.isCartIdxOnThisRank(global_index - 1)) + this->wbpData_[global_index] = 0.0; + } + forceDisableFipOutput_ = EWOMS_GET_PARAM(TypeTag, bool, ForceDisableFluidInPlaceOutput); } @@ -837,6 +842,8 @@ public: if (gasConnectionSaturations_.count(cartesianIdx) > 0) { gasConnectionSaturations_[cartesianIdx] = Opm::getValue(fs.saturation(gasPhaseIdx)); } + if (this->wbpData_.count(cartesianIdx) > 0) + this->wbpData_[cartesianIdx] = Opm::getValue(fs.pressure(oilPhaseIdx)); // tracers const auto& tracerModel = simulator_.problem().tracerModel(); @@ -1852,6 +1859,10 @@ public: return 0; } + const std::map& getWBPData() const { + return this->wbpData_; + } + const std::map, double>& getBlockData() { return blockData_; } @@ -2365,6 +2376,7 @@ private: ScalarBuffer hydrocarbonPoreVolume_; ScalarBuffer pressureTimesPoreVolume_; ScalarBuffer pressureTimesHydrocarbonVolume_; + std::map wbpData_; std::map, double> blockData_; std::map oilConnectionPressures_; std::map waterConnectionSaturations_; diff --git a/ebos/eclwriter.hh b/ebos/eclwriter.hh index 6291524be..06b6e1015 100644 --- a/ebos/eclwriter.hh +++ b/ebos/eclwriter.hh @@ -204,14 +204,12 @@ public: wbp_index_list = wbp_calculators.index_list(); } if (collectToIORank_.isParallel()) { -#ifdef HAVE_MPI + const auto& comm = simulator_.vanguard().grid().comm(); unsigned long size = wbp_index_list.size(); - MPI_Bcast(&size, 1, MPI_UNSIGNED_LONG, collectToIORank_.ioRank, MPI_COMM_WORLD); + comm.broadcast(&size, 1, collectToIORank_.ioRank); if (!collectToIORank_.isIORank()) wbp_index_list.resize( size ); - - MPI_Bcast(wbp_index_list.data(), size * sizeof(std::size_t), MPI_CHAR, collectToIORank_.ioRank, MPI_COMM_WORLD); -#endif + comm.broadcast(wbp_index_list.data(), size, collectToIORank_.ioRank); } // create output thread if enabled and rank is I/O rank // async output is enabled by default if pthread are enabled @@ -256,7 +254,6 @@ public: void evalSummaryState(bool isSubStep) { - PAvgCalculatorCollection wbp_calculators; const int reportStepNum = simulator_.episodeIndex() + 1; /* The summary data is not evaluated for timestep 0, that is @@ -294,8 +291,11 @@ public: this->prepareLocalCellData(isSubStep, reportStepNum); if (collectToIORank_.isParallel()) - collectToIORank_.collect({}, eclOutputModule_->getBlockData(), - localWellData, localGroupAndNetworkData); + collectToIORank_.collect({}, + eclOutputModule_->getBlockData(), + eclOutputModule_->getWBPData(), + localWellData, + localGroupAndNetworkData); std::map miscSummaryData; std::map> regionData; @@ -308,9 +308,18 @@ public: eclOutputModule_->outputInjLog(reportStepNum, isSubStep, forceDisableInjOutput); eclOutputModule_->outputCumLog(reportStepNum, isSubStep, forceDisableCumOutput); + std::vector buffer; if (this->collectToIORank_.isIORank()) { const auto& summary = eclIO_->summary(); + auto wbp_calculators = summary.wbp_calculators(reportStepNum); + const auto& wbpData + = this->collectToIORank_.isParallel() + ? this->collectToIORank_.globalWBPData() + : this->eclOutputModule_->getWBPData(); + + for (const auto& [global_index, pressure] : wbpData) + wbp_calculators.add_pressure( global_index, pressure ); // Add TCPU if (totalCpuTime != 0.0) { @@ -390,8 +399,11 @@ public: } if (this->collectToIORank_.isParallel()) { - collectToIORank_.collect(localCellData, eclOutputModule_->getBlockData(), - localWellData, localGroupAndNetworkData); + collectToIORank_.collect(localCellData, + eclOutputModule_->getBlockData(), + eclOutputModule_->getWBPData(), + localWellData, + localGroupAndNetworkData); } if (this->collectToIORank_.isIORank()) {