From 56bcb419fbf353f9fa6a8a11a9f20264284a783e Mon Sep 17 00:00:00 2001 From: Tor Harald Sandve Date: Thu, 1 Feb 2018 08:01:16 +0100 Subject: [PATCH] Add async output --- ebos/ecloutputblackoilmodule.hh | 3 + ebos/eclwriter.hh | 126 ++++++++++++++++++++++++++++---- 2 files changed, 115 insertions(+), 14 deletions(-) diff --git a/ebos/ecloutputblackoilmodule.hh b/ebos/ecloutputblackoilmodule.hh index 1c237fcbb..753e2520f 100644 --- a/ebos/ecloutputblackoilmodule.hh +++ b/ebos/ecloutputblackoilmodule.hh @@ -731,6 +731,9 @@ public: // write Fluid In Place to output log void outputFipLog(std::map& miscSummaryData, std::map>& regionData, const bool substep) { + if (substep) + return; + const auto& comm = simulator_.gridView().comm(); size_t ntFip = *std::max_element(fipnum_.begin(), fipnum_.end()); ntFip = comm.max(ntFip); diff --git a/ebos/eclwriter.hh b/ebos/eclwriter.hh index 61ab57f62..1e20236b1 100644 --- a/ebos/eclwriter.hh +++ b/ebos/eclwriter.hh @@ -33,7 +33,7 @@ #include #include - +#include #include #include @@ -48,6 +48,7 @@ #include #include #include +#include namespace Ewoms { namespace Properties { @@ -95,6 +96,7 @@ public: : simulator_(simulator) , collectToIORank_(simulator_.vanguard()) , eclOutputModule_(simulator, collectToIORank_) + , asyncOutput_() { globalGrid_ = simulator_.vanguard().grid(); globalGrid_.switchToGlobalView(); @@ -102,6 +104,25 @@ public: Opm::UgGridHelpers::createEclipseGrid( globalGrid_ , simulator_.vanguard().eclState().getInputGrid() ), simulator_.vanguard().schedule(), simulator_.vanguard().summaryConfig())); + + // create output thread if enabled and rank is I/O rank + // async output is enabled by default if pthread are enabled +#if HAVE_PTHREAD + const bool asyncOutputDefault = true; +#else + const bool asyncOutputDefault = false; +#endif + +// TODO Add param + const bool isIORank = collectToIORank_.isParallel() && collectToIORank_.isIORank(); + if( asyncOutputDefault && isIORank ) + { +#if HAVE_PTHREAD + asyncOutput_.reset( new Opm::ThreadHandle( isIORank ) ); +#else + throw std::runtime_error("Pthreads were not found, cannot enable async_output"); +#endif + } } ~EclWriter() @@ -127,7 +148,7 @@ public: /*! * \brief collect and pass data and pass it to eclIO writer */ - void writeOutput(const Opm::data::Wells& dw, Scalar t, bool substep, Scalar totalSolverTime, Scalar nextstep) + void writeOutput(const Opm::data::Wells& localWellData, Scalar t, bool substep, Scalar totalSolverTime, Scalar nextstep) { #if !HAVE_OPM_OUTPUT throw std::runtime_error("opm-output must be available to write ECL output!"); @@ -151,12 +172,12 @@ public: eclOutputModule_.outputErrorLog(); // collect all data to I/O rank and assign to sol - Opm::data::Solution localCellData; + Opm::data::Solution localCellData = {}; if (eclOutputModule_.outputRestart()) eclOutputModule_.assignToSolution(localCellData); if (collectToIORank_.isParallel()) - collectToIORank_.collect(localCellData, eclOutputModule_.getBlockValues()); + collectToIORank_.collect(localCellData, eclOutputModule_.getBlockValues(), localWellData); std::map miscSummaryData; std::map> regionData; @@ -177,18 +198,35 @@ public: } bool enableDoublePrecisionOutput = EWOMS_GET_PARAM(TypeTag, bool, EclOutputDoublePrecision); const Opm::data::Solution& cellData = collectToIORank_.isParallel() ? collectToIORank_.globalCellData() : localCellData; + const Opm::data::Wells& wellData = collectToIORank_.isParallel() ? collectToIORank_.globalWellData() : localWellData; + const std::map, double>& blockValues = collectToIORank_.isParallel() ? collectToIORank_.globalBlockValues() : eclOutputModule_.getBlockValues(); - eclIO_->writeTimeStep(episodeIdx, - substep, - t, - cellData, - dw, - miscSummaryData, - regionData, - blockValues, - extraRestartData, - enableDoublePrecisionOutput); + if( asyncOutput_ ) { + // dispatch the write call to the extra thread + asyncOutput_->dispatch( WriterCall(*eclIO_, + episodeIdx, + substep, + t, + cellData, + wellData, + miscSummaryData, + regionData, + blockValues, + extraRestartData, + enableDoublePrecisionOutput ) ); + } else { + eclIO_->writeTimeStep(episodeIdx, + substep, + t, + cellData, + wellData, + miscSummaryData, + regionData, + blockValues, + extraRestartData, + enableDoublePrecisionOutput); + } } #endif } @@ -385,6 +423,64 @@ private: return nnc; } + + struct WriterCall : public Opm::ThreadHandle :: ObjectInterface + { + Opm::EclipseIO& eclIO_; + int episodeIdx_; + bool isSubstep_; + double secondsElapsed_; + Opm::data::Solution cellData_; + Opm::data::Wells wellData_; + std::map singleSummaryValues_; + std::map> regionSummaryValues_; + std::map, double> blockSummaryValues_; + std::map> extraRestartData_; + bool writeDoublePrecision_; + + explicit WriterCall( + Opm::EclipseIO& eclIO, + int episodeIdx, + bool isSubstep, + double secondsElapsed, + Opm::data::Solution cellData, + Opm::data::Wells wellData, + const std::map& singleSummaryValues, + const std::map>& regionSummaryValues, + const std::map, double>& blockSummaryValues, + const std::map>& extraRestartData, + bool writeDoublePrecision) + : eclIO_(eclIO), + episodeIdx_(episodeIdx), + isSubstep_(isSubstep), + secondsElapsed_(secondsElapsed), + cellData_(cellData), + wellData_(wellData), + singleSummaryValues_(singleSummaryValues), + regionSummaryValues_(regionSummaryValues), + blockSummaryValues_(blockSummaryValues), + extraRestartData_(extraRestartData), + writeDoublePrecision_(writeDoublePrecision) + { + } + + // callback to eclIO serial writeTimeStep method + void run () + { + // write data + eclIO_.writeTimeStep(episodeIdx_, + isSubstep_, + secondsElapsed_, + cellData_, + wellData_, + singleSummaryValues_, + regionSummaryValues_, + blockSummaryValues_, + extraRestartData_, + writeDoublePrecision_); + } + }; + const Opm::EclipseState& eclState() const { return simulator_.vanguard().eclState(); } @@ -393,6 +489,8 @@ private: EclOutputBlackOilModule eclOutputModule_; std::unique_ptr eclIO_; Grid globalGrid_; + std::unique_ptr< Opm::ThreadHandle > asyncOutput_; + }; } // namespace Ewoms