Merge pull request #900 from blattms/fix-parallel-eclipse-writing

Fix using local active cells for writing eclipse files in parallel.

examples/sim_poly_fi2p_comp_ad.cpp

@@ -239,8 +239,13 @@ try
     std::cout << "\n\n================    Starting main simulation loop     ===============\n"
               << std::flush;
 
+    std::unique_ptr<Opm::EclipseWriter>
+        eclipseWriter(new Opm::EclipseWriter(*eclipseState,
+                                             UgGridHelpers
+                                             ::createEclipseGrid( cGrid ,
+                                                                  eclipseState->getInputGrid())));
     Opm::BlackoilOutputWriter
-        outputWriter(cGrid, param, *eclipseState, pu,
+        outputWriter(cGrid, param, *eclipseState, std::move(eclipseWriter), pu,
                      new_props->permeability() );
 
     SimulatorReport fullReport;

opm/autodiff/FlowMain.hpp

@@ -228,6 +228,7 @@ namespace Opm
         // distributeData()
         boost::any parallel_information_;
         // setupOutputWriter()
+        std::unique_ptr<EclipseWriter> eclipse_writer_;
         std::unique_ptr<BlackoilOutputWriter> output_writer_;
         // setupLinearSolver
         std::unique_ptr<NewtonIterationBlackoilInterface> fis_solver_;
@@ -754,9 +755,9 @@ namespace Opm
             if( output && output_ecl && output_cout_)
             {
                 const EclipseGrid& inputGrid = eclipse_state_->getInputGrid();
-                EclipseWriter writer(*eclipse_state_, UgGridHelpers::createEclipseGrid( grid , inputGrid ));
+                eclipse_writer_.reset(new EclipseWriter(*eclipse_state_, UgGridHelpers::createEclipseGrid( grid , inputGrid )));
-                writer.writeInitAndEgrid(geoprops_->simProps(grid),
+                eclipse_writer_->writeInitAndEgrid(geoprops_->simProps(grid),
-                                         geoprops_->nonCartesianConnections());
+                                                  geoprops_->nonCartesianConnections());
             }
         }
 
@@ -772,6 +773,7 @@ namespace Opm
             output_writer_.reset(new BlackoilOutputWriter(grid_init_->grid(),
                                                           param_,
                                                           *eclipse_state_,
+                                                          std::move(eclipse_writer_),
                                                           Opm::phaseUsageFromDeck(*deck_),
                                                           fluidprops_->permeability()));
         }

opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp

@@ -214,6 +214,7 @@ namespace Opm
         BlackoilOutputWriter(const Grid& grid,
                              const parameter::ParameterGroup& param,
                              const Opm::EclipseState& eclipseState,
+                             std::unique_ptr<EclipseWriter>&& eclWriter,
                              const Opm::PhaseUsage &phaseUsage,
                              const double* permeability );
 
@@ -328,6 +329,7 @@ namespace Opm
     BlackoilOutputWriter(const Grid& grid,
                          const parameter::ParameterGroup& param,
                          const Opm::EclipseState& eclipseState,
+                         std::unique_ptr<EclipseWriter>&& eclWriter,
                          const Opm::PhaseUsage &phaseUsage,
                          const double* permeability )
       : output_( param.getDefault("output", true) ),
@@ -336,49 +338,58 @@ namespace Opm
         output_interval_( output_ ? param.getDefault("output_interval", 1): 0 ),
         lastBackupReportStep_( -1 ),
         phaseUsage_( phaseUsage ),
-        vtkWriter_( output_ && param.getDefault("output_vtk",false) ?
-                     new BlackoilVTKWriter< Grid >( grid, outputDir_ ) : 0 ),
-        matlabWriter_( output_ && parallelOutput_->isIORank() &&
-                       param.getDefault("output_matlab", false) ?
-                     new BlackoilMatlabWriter< Grid >( grid, outputDir_ ) : 0 ),
-        eclWriter_( output_ && parallelOutput_->isIORank() &&
-                    param.getDefault("output_ecl", true) ?
-                    new EclipseWriter(eclipseState,UgGridHelpers::createEclipseGrid( grid , eclipseState.getInputGrid()))
-                   : 0 ),
         eclipseState_(eclipseState),
         asyncOutput_()
     {
         // For output.
-        if (output_ && parallelOutput_->isIORank() ) {
+        if ( output_ )
-            // Ensure that output dir exists
+        {
-            boost::filesystem::path fpath(outputDir_);
+            if ( param.getDefault("output_vtk",false) )
-            try {
+            {
-                create_directories(fpath);
+                vtkWriter_
-            }
+                    .reset(new BlackoilVTKWriter< Grid >( grid, outputDir_ ));
-            catch (...) {
-                OPM_THROW(std::runtime_error, "Creating directories failed: " << fpath);
             }
 
-            // create output thread if enabled and rank is I/O rank
+            if( parallelOutput_->isIORank() ) {
-            // async output is enabled by default if pthread are enabled
-#if HAVE_PTHREAD
-            const bool asyncOutputDefault = false;
-#else
-            const bool asyncOutputDefault = false;
-#endif
-            if( param.getDefault("async_output", asyncOutputDefault ) )
-            {
-#if HAVE_PTHREAD
-                asyncOutput_.reset( new ThreadHandle() );
-#else
-                OPM_THROW(std::runtime_error,"Pthreads were not found, cannot enable async_output");
-#endif
-            }
 
-            std::string backupfilename = param.getDefault("backupfile", std::string("") );
+                if ( param.getDefault("output_matlab", false ) )
-            if( ! backupfilename.empty() )
+                {
-            {
+                    matlabWriter_
-                backupfile_.open( backupfilename.c_str() );
+                        .reset(new BlackoilMatlabWriter< Grid >( grid, outputDir_ ));
+                }
+
+                eclWriter_ = std::move(eclWriter);
+
+                // Ensure that output dir exists
+                boost::filesystem::path fpath(outputDir_);
+                try {
+                    create_directories(fpath);
+                }
+                catch (...) {
+                    OPM_THROW(std::runtime_error, "Creating directories failed: " << fpath);
+                }
+
+                // create output thread if enabled and rank is I/O rank
+                // async output is enabled by default if pthread are enabled
+#if HAVE_PTHREAD
+                const bool asyncOutputDefault = false;
+#else
+                const bool asyncOutputDefault = false;
+#endif
+                if( param.getDefault("async_output", asyncOutputDefault ) )
+                {
+#if HAVE_PTHREAD
+                    asyncOutput_.reset( new ThreadHandle() );
+#else
+                    OPM_THROW(std::runtime_error,"Pthreads were not found, cannot enable async_output");
+#endif
+                }
+
+                std::string backupfilename = param.getDefault("backupfile", std::string("") );
+                if( ! backupfilename.empty() )
+                {
+                    backupfile_.open( backupfilename.c_str() );
+                }
             }
         }
     }
@@ -774,16 +785,56 @@ namespace Opm
                   const Model& physicalModel,
                   bool substep)
     {
+        data::Solution cellData{};
         const RestartConfig& restartConfig = eclipseState_.getRestartConfig();
         const SummaryConfig& summaryConfig = eclipseState_.getSummaryConfig();
         const int reportStepNum = timer.reportStepNum();
         bool logMessages = output_ && parallelOutput_->isIORank();
+        
+        if( output_ && !parallelOutput_->isParallel() )
+        {
 
-        data::Solution cellData;
+            detail::getRestartData( cellData, phaseUsage_, physicalModel,
-        detail::getRestartData( cellData, phaseUsage_, physicalModel,
+                                    restartConfig, reportStepNum, logMessages );
-                                restartConfig, reportStepNum, logMessages );
+            detail::getSummaryData( cellData, phaseUsage_, physicalModel, summaryConfig );
-        detail::getSummaryData( cellData, phaseUsage_, physicalModel, summaryConfig );
+        }
+        else
+        {
+            if ( logMessages )
+            {
+                std::map<std::string, int> rstKeywords = restartConfig.getRestartKeywords(reportStepNum);
+                std::vector<const char*> keywords = 
+                    { "WIP", "OIPL", "OIPG", "OIP", "GIPG", "GIPL", "GIP",
+                      "RPV", "FRPH", "RPRH"};
+                
+                std::ostringstream str;
+                str << "Output of restart/summary config not supported in parallel. Requested keywords were ";
+                std::size_t no_kw = 0;
+                    
+                auto func = [&] (const char* kw)
+                    {
+                        if ( detail::hasFRBKeyword(summaryConfig, kw) )
+                        {
+                            str << kw << " ";
+                            ++ no_kw;
+                        }
+                    };
 
+                std::for_each(keywords.begin(), keywords.end(), func);
+
+                for (auto& keyValue : rstKeywords)
+                {
+                        str << keyValue.first << " ";
+                        ++ no_kw;
+                }
+
+                if ( no_kw )
+                {
+                    Opm::OpmLog::warning("Unhandled ouput request", str.str());
+                }
+            }
+        }        
+        
         writeTimeStepWithCellProperties(timer, localState, localWellState, cellData, substep);
     }
 }