Merge pull request #484 from dr-robertk/PR/Parallel-output-only-in-parallel-runs

ParallelDebugOutput: only use parallel output when mpi size > 1.

examples/flow.cpp

@@ -348,7 +348,7 @@ try
     // create output writer after grid is distributed, otherwise the parallel output
     // won't work correctly since we need to create a mapping from the distributed to
     // the global view
-    Opm::BlackoilOutputWriter outputWriter(grid, param, eclipseState, pu );
+    Opm::BlackoilOutputWriter outputWriter(grid, param, eclipseState, pu, new_props.permeability() );
 
     // Solver for Newton iterations.
     std::unique_ptr<NewtonIterationBlackoilInterface> fis_solver;

examples/flow_solvent.cpp

@@ -358,7 +358,7 @@ try
     // create output writer after grid is distributed, otherwise the parallel output
     // won't work correctly since we need to create a mapping from the distributed to
     // the global view
-    Opm::BlackoilOutputWriter outputWriter(grid, param, eclipseState, pu );
+    Opm::BlackoilOutputWriter outputWriter(grid, param, eclipseState, pu, new_props.permeability() );
 
     // Solver for Newton iterations.
     std::unique_ptr<NewtonIterationBlackoilInterface> fis_solver;

opm/autodiff/ParallelDebugOutput.hpp

@@ -48,6 +48,7 @@ namespace Opm
         virtual const SimulatorState& globalReservoirState() const = 0 ;
         virtual const WellState& globalWellState() const = 0 ;
         virtual bool isIORank() const = 0;
+        virtual bool isParallel() const = 0;
         virtual int numCells() const = 0 ;
         virtual const int* globalCell() const = 0;
     };
@@ -64,7 +65,8 @@ namespace Opm
     public:
         ParallelDebugOutput ( const GridImpl& grid,
                               Opm::EclipseStateConstPtr /* eclipseState */,
-                              const int )
+                              const int,
+                              const double* )
             : grid_( grid ) {}
 
         // gather solution to rank 0 for EclipseWriter
@@ -79,6 +81,7 @@ namespace Opm
         virtual const SimulatorState& globalReservoirState() const { return *globalState_; }
         virtual const WellState& globalWellState() const { return *wellState_; }
         virtual bool isIORank () const { return true; }
+        virtual bool isParallel () const { return false; }
         virtual int numCells() const { return grid_.number_of_cells; }
         virtual const int* globalCell() const { return grid_.global_cell; }
     };
@@ -152,6 +155,7 @@ namespace Opm
                     globalPosition_.insert( std::make_pair( globalIndex[ index ], index ) );
                 }
 
+                // on I/O rank we need to create a mapping from local to global
                 if( ! indexMaps_.empty() )
                 {
                     // for the ioRank create a localIndex to index in global state map
@@ -161,7 +165,7 @@ namespace Opm
                     for( size_t i=0; i<localSize; ++i )
                     {
                         const int id = distributedGlobalIndex_[ localIndexMap_[ i ] ];
-                        indexMap[ i ] = id ;
+                        indexMap[ i ] = globalPosition_[ id ] ;
 #ifndef NDEBUG
                         assert( checkPosition_.find( id ) == checkPosition_.end() );
                         checkPosition_.insert( id );
@@ -216,11 +220,14 @@ namespace Opm
 
         ParallelDebugOutput( const Dune::CpGrid& otherGrid,
                              Opm::EclipseStateConstPtr eclipseState,
-                             const int numPhases )
+                             const int numPhases,
+                             const double* permeability )
             : toIORankComm_( otherGrid.comm() ),
               isIORank_( otherGrid.comm().rank() == ioRank )
         {
             const CollectiveCommunication& comm = otherGrid.comm();
+            if( comm.size() > 1 )
+            {
                 std::set< int > send, recv;
                 // the I/O rank receives from all other ranks
                 if( isIORank() )
@@ -241,8 +248,9 @@ namespace Opm
                                                Opm::UgGridHelpers::dimensions( globalGrid ),
                                                Opm::UgGridHelpers::cell2Faces( globalGrid ),
                                                Opm::UgGridHelpers::beginFaceCentroids( globalGrid ),
-                                           0,
+                                               permeability,
                                                false);
+
                     const Wells* wells = wells_manager.c_wells();
                     globalWellState_.init(wells, globalReservoirState_, globalWellState_ );
 
@@ -300,6 +308,12 @@ namespace Opm
                 DistributeIndexMapping distIndexMapping( globalIndex_, otherGrid.globalCell(), localIndexMap_, indexMaps_ );
                 toIORankComm_.exchange( distIndexMapping );
             }
+            else // serial run
+            {
+                // copy global cartesian index
+                globalIndex_ = otherGrid.globalCell();
+            }
+        }
 
         class PackUnPackSimulatorState : public P2PCommunicatorType::DataHandleInterface
         {
@@ -411,6 +425,7 @@ namespace Opm
                 for( unsigned int i=0; i<size; ++i )
                 {
                     const unsigned int index = localIndexMap[ i ] * stride + offset;
+                    assert( index < vector.size() );
                     buffer.write( vector[ index ] );
                 }
             }
@@ -427,6 +442,7 @@ namespace Opm
                 for( unsigned int i=0; i<size; ++i )
                 {
                     const unsigned int index = indexMap[ i ] * stride + offset;
+                    assert( index < vector.size() );
                     buffer.read( vector[ index ] );
                 }
             }
@@ -533,6 +549,11 @@ namespace Opm
             return isIORank_;
         }
 
+        bool isParallel() const
+        {
+            return toIORankComm_.size() > 1;
+        }
+
         int numCells () const { return globalIndex_.size(); }
         const int* globalCell () const
         {
@@ -545,7 +566,6 @@ namespace Opm
         IndexMapType                    globalIndex_;
         IndexMapType                    localIndexMap_;
         IndexMapStorageType             indexMaps_;
-        //BlackoilState                   globalReservoirState_;
         SimulatorState                  globalReservoirState_;
         // this needs to be revised
         WellStateFullyImplicitBlackoil  globalWellState_;

opm/autodiff/SimulatorFullyImplicitBlackoilOutput.cpp

@@ -261,18 +261,19 @@ namespace Opm
             vtkWriter_->writeTimeStep( timer, localState, localWellState, false );
         }
 
-        if( parallelOutput_ )
+        bool isIORank = true ;
+        if( parallelOutput_ && parallelOutput_->isParallel() )
         {
-
             // collect all solutions to I/O rank
-            const bool isIORank = parallelOutput_->collectToIORank( localState, localWellState );
+            isIORank = parallelOutput_->collectToIORank( localState, localWellState );
+        }
 
+        const SimulatorState& state = (parallelOutput_ && parallelOutput_->isParallel() ) ? parallelOutput_->globalReservoirState() : localState;
+        const WellState& wellState  = (parallelOutput_ && parallelOutput_->isParallel() ) ? parallelOutput_->globalWellState() : localWellState;
+
+        // output is only done on I/O rank
         if( isIORank )
         {
-                const SimulatorState& state = parallelOutput_->globalReservoirState();
-                const WellState& wellState  = parallelOutput_->globalWellState();
-                //std::cout << "number of wells" << wellState.bhp().size() << std::endl;
-
             // Matlab output
             if( matlabWriter_ ) {
                 matlabWriter_->writeTimeStep( timer, state, wellState, substep );
@@ -320,9 +321,8 @@ namespace Opm
                     */
                     backupfile_ << std::flush;
                 }
-                }
+            } // end backup
-            }
+        } // end isIORank
-        }
     }
 
     void

opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp

@@ -200,7 +200,8 @@ namespace Opm
         BlackoilOutputWriter(const Grid& grid,
                              const parameter::ParameterGroup& param,
                              Opm::EclipseStateConstPtr eclipseState,
-                             const Opm::PhaseUsage &phaseUsage);
+                             const Opm::PhaseUsage &phaseUsage,
+                             const double* permeability );
 
         /** \copydoc Opm::OutputWriter::writeInit */
         void writeInit(const SimulatorTimerInterface &timer);
@@ -251,9 +252,10 @@ namespace Opm
     BlackoilOutputWriter(const Grid& grid,
                          const parameter::ParameterGroup& param,
                          Opm::EclipseStateConstPtr eclipseState,
-                         const Opm::PhaseUsage &phaseUsage )
+                         const Opm::PhaseUsage &phaseUsage,
+                         const double* permeability )
       : output_( param.getDefault("output", true) ),
-        parallelOutput_( output_ ? new ParallelDebugOutput< Grid >( grid, eclipseState, phaseUsage.num_phases ) : 0 ),
+        parallelOutput_( output_ ? new ParallelDebugOutput< Grid >( grid, eclipseState, phaseUsage.num_phases, permeability ) : 0 ),
         outputDir_( output_ ? param.getDefault("output_dir", std::string("output")) : "." ),
         output_interval_( output_ ? param.getDefault("output_interval", 1): 0 ),
         lastBackupReportStep_( -1 ),