From 077dc02481a9bd3d1c227f216c19c3a1b91fa782 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Mon, 7 Nov 2016 19:35:53 +0100 Subject: [PATCH 1/4] Fix using local active cells for writing eclipse files in parallel. Previously, the eclipseGrid used by EclipseWriter was constructed from the one in the EclipseState with the current CpGrid. Unfortunately the latter was the distributed version resembling only the local part that the processor works on. Therefore the information about the active cells was wrong when writing results (which raised an exception in the writer). With this commit we construct the EclipseWriter before distributing the grid and use this writer later on in the OutputWriter. --- examples/sim_poly_fi2p_comp_ad.cpp | 7 ++++++- opm/autodiff/FlowMain.hpp | 8 +++++--- opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp | 4 +++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/examples/sim_poly_fi2p_comp_ad.cpp b/examples/sim_poly_fi2p_comp_ad.cpp index 17bc8bfa6..e1d4d77fb 100644 --- a/examples/sim_poly_fi2p_comp_ad.cpp +++ b/examples/sim_poly_fi2p_comp_ad.cpp @@ -239,8 +239,13 @@ try std::cout << "\n\n================ Starting main simulation loop ===============\n" << std::flush; + std::unique_ptr + eclipseWriter(new Opm::EclipseWriter(*eclipseState, + UgGridHelpers + ::createEclipseGrid( cGrid , + eclipseState->getInputGrid()))); Opm::BlackoilOutputWriter - outputWriter(cGrid, param, *eclipseState, pu, + outputWriter(cGrid, param, *eclipseState, eclipseWriter, pu, new_props->permeability() ); SimulatorReport fullReport; diff --git a/opm/autodiff/FlowMain.hpp b/opm/autodiff/FlowMain.hpp index 95092e7e2..3ac00166e 100644 --- a/opm/autodiff/FlowMain.hpp +++ b/opm/autodiff/FlowMain.hpp @@ -228,6 +228,7 @@ namespace Opm // distributeData() boost::any parallel_information_; // setupOutputWriter() + std::unique_ptr eclipse_writer_; std::unique_ptr output_writer_; // setupLinearSolver std::unique_ptr fis_solver_; @@ -754,9 +755,9 @@ namespace Opm if( output && output_ecl && output_cout_) { const EclipseGrid& inputGrid = eclipse_state_->getInputGrid(); - EclipseWriter writer(*eclipse_state_, UgGridHelpers::createEclipseGrid( grid , inputGrid )); - writer.writeInitAndEgrid(geoprops_->simProps(grid), - geoprops_->nonCartesianConnections()); + eclipse_writer_.reset(new EclipseWriter(*eclipse_state_, UgGridHelpers::createEclipseGrid( grid , inputGrid ))); + eclipse_writer_->writeInitAndEgrid(geoprops_->simProps(grid), + geoprops_->nonCartesianConnections()); } } @@ -772,6 +773,7 @@ namespace Opm output_writer_.reset(new BlackoilOutputWriter(grid_init_->grid(), param_, *eclipse_state_, + eclipse_writer_, Opm::phaseUsageFromDeck(*deck_), fluidprops_->permeability())); } diff --git a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp index 4348b9a26..7f7cef102 100644 --- a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp +++ b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp @@ -214,6 +214,7 @@ namespace Opm BlackoilOutputWriter(const Grid& grid, const parameter::ParameterGroup& param, const Opm::EclipseState& eclipseState, + std::unique_ptr& eclWriter, const Opm::PhaseUsage &phaseUsage, const double* permeability ); @@ -328,6 +329,7 @@ namespace Opm BlackoilOutputWriter(const Grid& grid, const parameter::ParameterGroup& param, const Opm::EclipseState& eclipseState, + std::unique_ptr& eclWriter, const Opm::PhaseUsage &phaseUsage, const double* permeability ) : output_( param.getDefault("output", true) ), @@ -343,7 +345,7 @@ namespace Opm new BlackoilMatlabWriter< Grid >( grid, outputDir_ ) : 0 ), eclWriter_( output_ && parallelOutput_->isIORank() && param.getDefault("output_ecl", true) ? - new EclipseWriter(eclipseState,UgGridHelpers::createEclipseGrid( grid , eclipseState.getInputGrid())) + eclWriter.release() : 0 ), eclipseState_(eclipseState), asyncOutput_() From 3eba3353d0b9f0ef93a4bec579177e8f6b17f7f9 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Thu, 10 Nov 2016 12:41:13 +0100 Subject: [PATCH 2/4] Correctly mark transfer of ownership for ouptut writer --- examples/sim_poly_fi2p_comp_ad.cpp | 2 +- opm/autodiff/FlowMain.hpp | 2 +- opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/sim_poly_fi2p_comp_ad.cpp b/examples/sim_poly_fi2p_comp_ad.cpp index e1d4d77fb..2be72276d 100644 --- a/examples/sim_poly_fi2p_comp_ad.cpp +++ b/examples/sim_poly_fi2p_comp_ad.cpp @@ -245,7 +245,7 @@ try ::createEclipseGrid( cGrid , eclipseState->getInputGrid()))); Opm::BlackoilOutputWriter - outputWriter(cGrid, param, *eclipseState, eclipseWriter, pu, + outputWriter(cGrid, param, *eclipseState, std::move(eclipseWriter), pu, new_props->permeability() ); SimulatorReport fullReport; diff --git a/opm/autodiff/FlowMain.hpp b/opm/autodiff/FlowMain.hpp index 3ac00166e..a254fd033 100644 --- a/opm/autodiff/FlowMain.hpp +++ b/opm/autodiff/FlowMain.hpp @@ -773,7 +773,7 @@ namespace Opm output_writer_.reset(new BlackoilOutputWriter(grid_init_->grid(), param_, *eclipse_state_, - eclipse_writer_, + std::move(eclipse_writer_), Opm::phaseUsageFromDeck(*deck_), fluidprops_->permeability())); } diff --git a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp index 7f7cef102..ea0dffb40 100644 --- a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp +++ b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp @@ -214,7 +214,7 @@ namespace Opm BlackoilOutputWriter(const Grid& grid, const parameter::ParameterGroup& param, const Opm::EclipseState& eclipseState, - std::unique_ptr& eclWriter, + std::unique_ptr&& eclWriter, const Opm::PhaseUsage &phaseUsage, const double* permeability ); @@ -329,7 +329,7 @@ namespace Opm BlackoilOutputWriter(const Grid& grid, const parameter::ParameterGroup& param, const Opm::EclipseState& eclipseState, - std::unique_ptr& eclWriter, + std::unique_ptr&& eclWriter, const Opm::PhaseUsage &phaseUsage, const double* permeability ) : output_( param.getDefault("output", true) ), @@ -345,8 +345,8 @@ namespace Opm new BlackoilMatlabWriter< Grid >( grid, outputDir_ ) : 0 ), eclWriter_( output_ && parallelOutput_->isIORank() && param.getDefault("output_ecl", true) ? - eclWriter.release() - : 0 ), + std::move(eclWriter) + : std::move(std::unique_ptr()) ), eclipseState_(eclipseState), asyncOutput_() { From 747c93e898d9325d3d2879c0e85de9200c4bd2a5 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Thu, 10 Nov 2016 13:28:21 +0100 Subject: [PATCH 3/4] Removed ternary operator in inline initialization. This made the code rather hard to read. Now we do the initialization in the constructor body if needed. --- .../SimulatorFullyImplicitBlackoilOutput.hpp | 81 ++++++++++--------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp index ea0dffb40..ef879d17f 100644 --- a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp +++ b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp @@ -338,49 +338,58 @@ namespace Opm output_interval_( output_ ? param.getDefault("output_interval", 1): 0 ), lastBackupReportStep_( -1 ), phaseUsage_( phaseUsage ), - vtkWriter_( output_ && param.getDefault("output_vtk",false) ? - new BlackoilVTKWriter< Grid >( grid, outputDir_ ) : 0 ), - matlabWriter_( output_ && parallelOutput_->isIORank() && - param.getDefault("output_matlab", false) ? - new BlackoilMatlabWriter< Grid >( grid, outputDir_ ) : 0 ), - eclWriter_( output_ && parallelOutput_->isIORank() && - param.getDefault("output_ecl", true) ? - std::move(eclWriter) - : std::move(std::unique_ptr()) ), eclipseState_(eclipseState), asyncOutput_() { // For output. - if (output_ && parallelOutput_->isIORank() ) { - // Ensure that output dir exists - boost::filesystem::path fpath(outputDir_); - try { - create_directories(fpath); - } - catch (...) { - OPM_THROW(std::runtime_error, "Creating directories failed: " << fpath); + if ( output_ ) + { + if ( param.getDefault("output_vtk",false) ) + { + vtkWriter_ + .reset(new BlackoilVTKWriter< Grid >( grid, outputDir_ )); } - // create output thread if enabled and rank is I/O rank - // async output is enabled by default if pthread are enabled -#if HAVE_PTHREAD - const bool asyncOutputDefault = false; -#else - const bool asyncOutputDefault = false; -#endif - if( param.getDefault("async_output", asyncOutputDefault ) ) - { -#if HAVE_PTHREAD - asyncOutput_.reset( new ThreadHandle() ); -#else - OPM_THROW(std::runtime_error,"Pthreads were not found, cannot enable async_output"); -#endif - } + if( parallelOutput_->isIORank() ) { - std::string backupfilename = param.getDefault("backupfile", std::string("") ); - if( ! backupfilename.empty() ) - { - backupfile_.open( backupfilename.c_str() ); + if ( param.getDefault("output_matlab", false ) ) + { + matlabWriter_ + .reset(new BlackoilMatlabWriter< Grid >( grid, outputDir_ )); + } + + eclWriter_ = std::move(eclWriter); + + // Ensure that output dir exists + boost::filesystem::path fpath(outputDir_); + try { + create_directories(fpath); + } + catch (...) { + OPM_THROW(std::runtime_error, "Creating directories failed: " << fpath); + } + + // create output thread if enabled and rank is I/O rank + // async output is enabled by default if pthread are enabled +#if HAVE_PTHREAD + const bool asyncOutputDefault = false; +#else + const bool asyncOutputDefault = false; +#endif + if( param.getDefault("async_output", asyncOutputDefault ) ) + { +#if HAVE_PTHREAD + asyncOutput_.reset( new ThreadHandle() ); +#else + OPM_THROW(std::runtime_error,"Pthreads were not found, cannot enable async_output"); +#endif + } + + std::string backupfilename = param.getDefault("backupfile", std::string("") ); + if( ! backupfilename.empty() ) + { + backupfile_.open( backupfilename.c_str() ); + } } } } From 8632c71cefe039a9ec3569d391c4c9baa24056fa Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Thu, 10 Nov 2016 17:46:49 +0100 Subject: [PATCH 4/4] Do not populate cellData but issue a warning in parallel. This will lead to some data missing in the parallel runs but will allow successful runs. --- .../SimulatorFullyImplicitBlackoilOutput.hpp | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp index ef879d17f..d11b0c7fb 100644 --- a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp +++ b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp @@ -785,16 +785,56 @@ namespace Opm const Model& physicalModel, bool substep) { + data::Solution cellData{}; const RestartConfig& restartConfig = eclipseState_.getRestartConfig(); const SummaryConfig& summaryConfig = eclipseState_.getSummaryConfig(); const int reportStepNum = timer.reportStepNum(); bool logMessages = output_ && parallelOutput_->isIORank(); + + if( output_ && !parallelOutput_->isParallel() ) + { - data::Solution cellData; - detail::getRestartData( cellData, phaseUsage_, physicalModel, - restartConfig, reportStepNum, logMessages ); - detail::getSummaryData( cellData, phaseUsage_, physicalModel, summaryConfig ); + detail::getRestartData( cellData, phaseUsage_, physicalModel, + restartConfig, reportStepNum, logMessages ); + detail::getSummaryData( cellData, phaseUsage_, physicalModel, summaryConfig ); + } + else + { + if ( logMessages ) + { + std::map rstKeywords = restartConfig.getRestartKeywords(reportStepNum); + std::vector keywords = + { "WIP", "OIPL", "OIPG", "OIP", "GIPG", "GIPL", "GIP", + "RPV", "FRPH", "RPRH"}; + + std::ostringstream str; + str << "Output of restart/summary config not supported in parallel. Requested keywords were "; + std::size_t no_kw = 0; + + auto func = [&] (const char* kw) + { + if ( detail::hasFRBKeyword(summaryConfig, kw) ) + { + str << kw << " "; + ++ no_kw; + } + }; + std::for_each(keywords.begin(), keywords.end(), func); + + for (auto& keyValue : rstKeywords) + { + str << keyValue.first << " "; + ++ no_kw; + } + + if ( no_kw ) + { + Opm::OpmLog::warning("Unhandled ouput request", str.str()); + } + } + } + writeTimeStepWithCellProperties(timer, localState, localWellState, cellData, substep); } }