From dacb4013eda83d9baf9acb85a91e8774f3924a68 Mon Sep 17 00:00:00 2001 From: Robert Kloefkorn Date: Mon, 21 Sep 2015 12:24:07 +0200 Subject: [PATCH 1/5] ParallelDebugOutput: only use parallel output when mpi size > 1. --- opm/autodiff/ParallelDebugOutput.hpp | 149 ++++++++++-------- .../SimulatorFullyImplicitBlackoilOutput.cpp | 118 +++++++------- 2 files changed, 138 insertions(+), 129 deletions(-) diff --git a/opm/autodiff/ParallelDebugOutput.hpp b/opm/autodiff/ParallelDebugOutput.hpp index 45d41a492..7671e3789 100644 --- a/opm/autodiff/ParallelDebugOutput.hpp +++ b/opm/autodiff/ParallelDebugOutput.hpp @@ -48,6 +48,7 @@ namespace Opm virtual const SimulatorState& globalReservoirState() const = 0 ; virtual const WellState& globalWellState() const = 0 ; virtual bool isIORank() const = 0; + virtual bool isParallel() const = 0; virtual int numCells() const = 0 ; virtual const int* globalCell() const = 0; }; @@ -79,6 +80,7 @@ namespace Opm virtual const SimulatorState& globalReservoirState() const { return *globalState_; } virtual const WellState& globalWellState() const { return *wellState_; } virtual bool isIORank () const { return true; } + virtual bool isParallel () const { return false; } virtual int numCells() const { return grid_.number_of_cells; } virtual const int* globalCell() const { return grid_.global_cell; } }; @@ -221,84 +223,87 @@ namespace Opm isIORank_( otherGrid.comm().rank() == ioRank ) { const CollectiveCommunication& comm = otherGrid.comm(); - std::set< int > send, recv; - // the I/O rank receives from all other ranks - if( isIORank() ) + if( comm.size() > 1 ) { - Dune::CpGrid globalGrid( otherGrid ); - globalGrid.switchToGlobalView(); - - // initialize global state with correct sizes - globalReservoirState_.init( globalGrid.numCells(), globalGrid.numFaces(), numPhases ); - // TODO init well state - - // Create wells and well state. - WellsManager wells_manager(eclipseState, - 0, - Opm::UgGridHelpers::numCells( globalGrid ), - Opm::UgGridHelpers::globalCell( globalGrid ), - Opm::UgGridHelpers::cartDims( globalGrid ), - Opm::UgGridHelpers::dimensions( globalGrid ), - Opm::UgGridHelpers::cell2Faces( globalGrid ), - Opm::UgGridHelpers::beginFaceCentroids( globalGrid ), - 0, - false); - const Wells* wells = wells_manager.c_wells(); - globalWellState_.init(wells, globalReservoirState_, globalWellState_ ); - - // copy global cartesian index - globalIndex_ = globalGrid.globalCell(); - - unsigned int count = 0; - auto gridView = globalGrid.leafGridView(); - for( auto it = gridView.begin< 0 >(), - end = gridView.end< 0 >(); it != end; ++it, ++count ) + std::set< int > send, recv; + // the I/O rank receives from all other ranks + if( isIORank() ) { - } - assert( count == globalIndex_.size() ); + Dune::CpGrid globalGrid( otherGrid ); + globalGrid.switchToGlobalView(); - for(int i=0; i(), + end = gridView.end< 0 >(); it != end; ++it, ++count ) { - recv.insert( i ); + } + assert( count == globalIndex_.size() ); + + for(int i=0; i(), - end = localView.end< 0 >(); it != end; ++it, ++index ) - { - const auto element = *it ; - // only store interior element for collection - if( element.partitionType() == Dune :: InteriorEntity ) + else // all other simply send to the I/O rank { - localIndexMap_.push_back( index ); + send.insert( ioRank ); } + + localIndexMap_.clear(); + localIndexMap_.reserve( otherGrid.size( 0 ) ); + + unsigned int index = 0; + auto localView = otherGrid.leafGridView(); + for( auto it = localView.begin< 0 >(), + end = localView.end< 0 >(); it != end; ++it, ++index ) + { + const auto element = *it ; + // only store interior element for collection + if( element.partitionType() == Dune :: InteriorEntity ) + { + localIndexMap_.push_back( index ); + } + } + + // insert send and recv linkage to communicator + toIORankComm_.insertRequest( send, recv ); + + if( isIORank() ) + { + // need an index map for each rank + indexMaps_.clear(); + indexMaps_.resize( comm.size() ); + } + + // distribute global id's to io rank for later association of dof's + DistributeIndexMapping distIndexMapping( globalIndex_, otherGrid.globalCell(), localIndexMap_, indexMaps_ ); + toIORankComm_.exchange( distIndexMapping ); } - - // insert send and recv linkage to communicator - toIORankComm_.insertRequest( send, recv ); - - if( isIORank() ) - { - // need an index map for each rank - indexMaps_.clear(); - indexMaps_.resize( comm.size() ); - } - - // distribute global id's to io rank for later association of dof's - DistributeIndexMapping distIndexMapping( globalIndex_, otherGrid.globalCell(), localIndexMap_, indexMaps_ ); - toIORankComm_.exchange( distIndexMapping ); } class PackUnPackSimulatorState : public P2PCommunicatorType::DataHandleInterface @@ -533,6 +538,11 @@ namespace Opm return isIORank_; } + bool isParallel() const + { + return toIORankComm_.size() > 1; + } + int numCells () const { return globalIndex_.size(); } const int* globalCell () const { @@ -545,7 +555,6 @@ namespace Opm IndexMapType globalIndex_; IndexMapType localIndexMap_; IndexMapStorageType indexMaps_; - //BlackoilState globalReservoirState_; SimulatorState globalReservoirState_; // this needs to be revised WellStateFullyImplicitBlackoil globalWellState_; diff --git a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.cpp b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.cpp index 61b8c052c..c694b1980 100644 --- a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.cpp +++ b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.cpp @@ -261,68 +261,68 @@ namespace Opm vtkWriter_->writeTimeStep( timer, localState, localWellState, false ); } - if( parallelOutput_ ) + bool isIORank = true ; + if( parallelOutput_ && parallelOutput_->isParallel() ) { - // collect all solutions to I/O rank - const bool isIORank = parallelOutput_->collectToIORank( localState, localWellState ); - - if( isIORank ) - { - const SimulatorState& state = parallelOutput_->globalReservoirState(); - const WellState& wellState = parallelOutput_->globalWellState(); - //std::cout << "number of wells" << wellState.bhp().size() << std::endl; - - // Matlab output - if( matlabWriter_ ) { - matlabWriter_->writeTimeStep( timer, state, wellState, substep ); - } - // ECL output - if ( eclWriter_ ) { - eclWriter_->writeTimeStep(timer, state, wellState, substep ); - } - - // write backup file - if( backupfile_ ) - { - int reportStep = timer.reportStepNum(); - int currentTimeStep = timer.currentStepNum(); - if( (reportStep == currentTimeStep || // true for SimulatorTimer - currentTimeStep == 0 || // true for AdaptiveSimulatorTimer at reportStep - timer.done() ) // true for AdaptiveSimulatorTimer at reportStep - && lastBackupReportStep_ != reportStep ) // only backup report step once - { - // store report step - lastBackupReportStep_ = reportStep; - // write resport step number - backupfile_.write( (const char *) &reportStep, sizeof(int) ); - - try { - const BlackoilState& boState = dynamic_cast< const BlackoilState& > (state); - backupfile_ << boState; - - const WellStateFullyImplicitBlackoil& boWellState = static_cast< const WellStateFullyImplicitBlackoil& > (wellState); - backupfile_ << boWellState; - } - catch ( const std::bad_cast& e ) - { - - } - - /* - const WellStateFullyImplicitBlackoil* boWellState = - dynamic_cast< const WellStateFullyImplicitBlackoil* > (&wellState); - if( boWellState ) { - backupfile_ << (*boWellState); - } - else - OPM_THROW(std::logic_error,"cast to WellStateFullyImplicitBlackoil failed"); - */ - backupfile_ << std::flush; - } - } - } + isIORank = parallelOutput_->collectToIORank( localState, localWellState ); } + + const SimulatorState& state = (parallelOutput_ && parallelOutput_->isParallel() ) ? parallelOutput_->globalReservoirState() : localState; + const WellState& wellState = (parallelOutput_ && parallelOutput_->isParallel() ) ? parallelOutput_->globalWellState() : localWellState; + + // output is only done on I/O rank + if( isIORank ) + { + // Matlab output + if( matlabWriter_ ) { + matlabWriter_->writeTimeStep( timer, state, wellState, substep ); + } + // ECL output + if ( eclWriter_ ) { + eclWriter_->writeTimeStep(timer, state, wellState, substep ); + } + + // write backup file + if( backupfile_ ) + { + int reportStep = timer.reportStepNum(); + int currentTimeStep = timer.currentStepNum(); + if( (reportStep == currentTimeStep || // true for SimulatorTimer + currentTimeStep == 0 || // true for AdaptiveSimulatorTimer at reportStep + timer.done() ) // true for AdaptiveSimulatorTimer at reportStep + && lastBackupReportStep_ != reportStep ) // only backup report step once + { + // store report step + lastBackupReportStep_ = reportStep; + // write resport step number + backupfile_.write( (const char *) &reportStep, sizeof(int) ); + + try { + const BlackoilState& boState = dynamic_cast< const BlackoilState& > (state); + backupfile_ << boState; + + const WellStateFullyImplicitBlackoil& boWellState = static_cast< const WellStateFullyImplicitBlackoil& > (wellState); + backupfile_ << boWellState; + } + catch ( const std::bad_cast& e ) + { + + } + + /* + const WellStateFullyImplicitBlackoil* boWellState = + dynamic_cast< const WellStateFullyImplicitBlackoil* > (&wellState); + if( boWellState ) { + backupfile_ << (*boWellState); + } + else + OPM_THROW(std::logic_error,"cast to WellStateFullyImplicitBlackoil failed"); + */ + backupfile_ << std::flush; + } + } // end backup + } // end isIORank } void From f63487ad0ff5f97628401703e0ab7d5f8fb0939c Mon Sep 17 00:00:00 2001 From: Robert Kloefkorn Date: Mon, 21 Sep 2015 12:42:41 +0200 Subject: [PATCH 2/5] ParallelDebugOutput: initialize globalIndex in serial run too. --- opm/autodiff/ParallelDebugOutput.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/opm/autodiff/ParallelDebugOutput.hpp b/opm/autodiff/ParallelDebugOutput.hpp index 7671e3789..b4c7c1f79 100644 --- a/opm/autodiff/ParallelDebugOutput.hpp +++ b/opm/autodiff/ParallelDebugOutput.hpp @@ -304,6 +304,11 @@ namespace Opm DistributeIndexMapping distIndexMapping( globalIndex_, otherGrid.globalCell(), localIndexMap_, indexMaps_ ); toIORankComm_.exchange( distIndexMapping ); } + else // serial run + { + // copy global cartesian index + globalIndex_ = otherGrid.globalCell(); + } } class PackUnPackSimulatorState : public P2PCommunicatorType::DataHandleInterface From 04ec4eb2bf81155344a3193e4854ac08e4325109 Mon Sep 17 00:00:00 2001 From: Robert Kloefkorn Date: Mon, 21 Sep 2015 12:52:37 +0200 Subject: [PATCH 3/5] ParallelDebugOutput: disable wells, seems to not work after latest changes in opm-core. --- opm/autodiff/ParallelDebugOutput.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/opm/autodiff/ParallelDebugOutput.hpp b/opm/autodiff/ParallelDebugOutput.hpp index b4c7c1f79..64d88b255 100644 --- a/opm/autodiff/ParallelDebugOutput.hpp +++ b/opm/autodiff/ParallelDebugOutput.hpp @@ -236,6 +236,7 @@ namespace Opm globalReservoirState_.init( globalGrid.numCells(), globalGrid.numFaces(), numPhases ); // TODO init well state + /* // Create wells and well state. WellsManager wells_manager(eclipseState, 0, @@ -247,8 +248,10 @@ namespace Opm Opm::UgGridHelpers::beginFaceCentroids( globalGrid ), 0, false); + const Wells* wells = wells_manager.c_wells(); globalWellState_.init(wells, globalReservoirState_, globalWellState_ ); + */ // copy global cartesian index globalIndex_ = globalGrid.globalCell(); @@ -378,7 +381,7 @@ namespace Opm } // write all data from local well state to buffer - writeWells( buffer ); + // writeWells( buffer ); } void doUnpack( const IndexMapType& indexMap, MessageBufferType& buffer ) @@ -400,7 +403,7 @@ namespace Opm } // read well data from buffer - readWells( buffer ); + // readWells( buffer ); } // unpack all data associated with link From 299b0aa3cf96c36f70e3f225acfc077ded8c4d7b Mon Sep 17 00:00:00 2001 From: Robert Kloefkorn Date: Mon, 21 Sep 2015 14:05:35 +0200 Subject: [PATCH 4/5] ParallelDebugOutput: fix initialization of wells for the I/O rank. --- examples/flow.cpp | 2 +- examples/flow_solvent.cpp | 2 +- opm/autodiff/ParallelDebugOutput.hpp | 14 +++++++------- .../SimulatorFullyImplicitBlackoilOutput.hpp | 8 +++++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/examples/flow.cpp b/examples/flow.cpp index 7ccd5016e..7c9dea376 100644 --- a/examples/flow.cpp +++ b/examples/flow.cpp @@ -348,7 +348,7 @@ try // create output writer after grid is distributed, otherwise the parallel output // won't work correctly since we need to create a mapping from the distributed to // the global view - Opm::BlackoilOutputWriter outputWriter(grid, param, eclipseState, pu ); + Opm::BlackoilOutputWriter outputWriter(grid, param, eclipseState, pu, new_props.permeability() ); // Solver for Newton iterations. std::unique_ptr fis_solver; diff --git a/examples/flow_solvent.cpp b/examples/flow_solvent.cpp index 48909c19e..aa576810d 100644 --- a/examples/flow_solvent.cpp +++ b/examples/flow_solvent.cpp @@ -358,7 +358,7 @@ try // create output writer after grid is distributed, otherwise the parallel output // won't work correctly since we need to create a mapping from the distributed to // the global view - Opm::BlackoilOutputWriter outputWriter(grid, param, eclipseState, pu ); + Opm::BlackoilOutputWriter outputWriter(grid, param, eclipseState, pu, new_props.permeability() ); // Solver for Newton iterations. std::unique_ptr fis_solver; diff --git a/opm/autodiff/ParallelDebugOutput.hpp b/opm/autodiff/ParallelDebugOutput.hpp index 64d88b255..d2617dc02 100644 --- a/opm/autodiff/ParallelDebugOutput.hpp +++ b/opm/autodiff/ParallelDebugOutput.hpp @@ -65,7 +65,8 @@ namespace Opm public: ParallelDebugOutput ( const GridImpl& grid, Opm::EclipseStateConstPtr /* eclipseState */, - const int ) + const int, + const double* ) : grid_( grid ) {} // gather solution to rank 0 for EclipseWriter @@ -218,7 +219,8 @@ namespace Opm ParallelDebugOutput( const Dune::CpGrid& otherGrid, Opm::EclipseStateConstPtr eclipseState, - const int numPhases ) + const int numPhases, + const double* permeability ) : toIORankComm_( otherGrid.comm() ), isIORank_( otherGrid.comm().rank() == ioRank ) { @@ -236,7 +238,6 @@ namespace Opm globalReservoirState_.init( globalGrid.numCells(), globalGrid.numFaces(), numPhases ); // TODO init well state - /* // Create wells and well state. WellsManager wells_manager(eclipseState, 0, @@ -246,12 +247,11 @@ namespace Opm Opm::UgGridHelpers::dimensions( globalGrid ), Opm::UgGridHelpers::cell2Faces( globalGrid ), Opm::UgGridHelpers::beginFaceCentroids( globalGrid ), - 0, + permeability, false); const Wells* wells = wells_manager.c_wells(); globalWellState_.init(wells, globalReservoirState_, globalWellState_ ); - */ // copy global cartesian index globalIndex_ = globalGrid.globalCell(); @@ -381,7 +381,7 @@ namespace Opm } // write all data from local well state to buffer - // writeWells( buffer ); + writeWells( buffer ); } void doUnpack( const IndexMapType& indexMap, MessageBufferType& buffer ) @@ -403,7 +403,7 @@ namespace Opm } // read well data from buffer - // readWells( buffer ); + readWells( buffer ); } // unpack all data associated with link diff --git a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp index d70461450..8de66c6bc 100644 --- a/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp +++ b/opm/autodiff/SimulatorFullyImplicitBlackoilOutput.hpp @@ -200,7 +200,8 @@ namespace Opm BlackoilOutputWriter(const Grid& grid, const parameter::ParameterGroup& param, Opm::EclipseStateConstPtr eclipseState, - const Opm::PhaseUsage &phaseUsage); + const Opm::PhaseUsage &phaseUsage, + const double* permeability ); /** \copydoc Opm::OutputWriter::writeInit */ void writeInit(const SimulatorTimerInterface &timer); @@ -251,9 +252,10 @@ namespace Opm BlackoilOutputWriter(const Grid& grid, const parameter::ParameterGroup& param, Opm::EclipseStateConstPtr eclipseState, - const Opm::PhaseUsage &phaseUsage ) + const Opm::PhaseUsage &phaseUsage, + const double* permeability ) : output_( param.getDefault("output", true) ), - parallelOutput_( output_ ? new ParallelDebugOutput< Grid >( grid, eclipseState, phaseUsage.num_phases ) : 0 ), + parallelOutput_( output_ ? new ParallelDebugOutput< Grid >( grid, eclipseState, phaseUsage.num_phases, permeability ) : 0 ), outputDir_( output_ ? param.getDefault("output_dir", std::string("output")) : "." ), output_interval_( output_ ? param.getDefault("output_interval", 1): 0 ), lastBackupReportStep_( -1 ), From 36ca75c4b724b850d6d3375c3f9234eceda27ad7 Mon Sep 17 00:00:00 2001 From: Robert Kloefkorn Date: Mon, 21 Sep 2015 16:48:57 +0200 Subject: [PATCH 5/5] ParallelDebugOutput: fix bug in creation of local to global map on I/O rank. --- opm/autodiff/ParallelDebugOutput.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/opm/autodiff/ParallelDebugOutput.hpp b/opm/autodiff/ParallelDebugOutput.hpp index d2617dc02..fe317e629 100644 --- a/opm/autodiff/ParallelDebugOutput.hpp +++ b/opm/autodiff/ParallelDebugOutput.hpp @@ -155,6 +155,7 @@ namespace Opm globalPosition_.insert( std::make_pair( globalIndex[ index ], index ) ); } + // on I/O rank we need to create a mapping from local to global if( ! indexMaps_.empty() ) { // for the ioRank create a localIndex to index in global state map @@ -164,7 +165,7 @@ namespace Opm for( size_t i=0; i