OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Adds data redistribution capabilities and makes sim_fibo_ad_cp parallel.

Browse Source 
With this commit we add the possibility to start with a global representation
of a simulator that is read on each process and afterwards this presentation
is redistributed among the processors together with the properties and
state data needed to initialize the simulation.

There still is no parallel well handling and no parallel output. But with the
equilibrium example of @dr-robertk and deactivated output we can already
perform parallel runs.
This commit is contained in:
Markus Blatt 2015-02-05 15:05:38 +01:00
parent fc137afcd5
commit 01ea7bacba
6 changed files with 271 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists_files.cmake
View File

@ -113,6 +113,7 @@ list (APPEND PUBLIC_HEADER_FILES
opm/autodiff/NewtonIterationBlackoilSimple.hpp
opm/autodiff/LinearisedBlackoilResidual.hpp
opm/autodiff/RateConverter.hpp
opm/autodiff/RedistributeDataHandles.hpp
opm/autodiff/SimulatorCompressibleAd.hpp
opm/autodiff/SimulatorFullyImplicitBlackoil.hpp
opm/autodiff/SimulatorFullyImplicitBlackoil_impl.hpp

47
examples/sim_fibo_ad_cp.cpp
View File

@ -68,6 +68,7 @@
#include <opm/autodiff/SimulatorFullyImplicitBlackoil.hpp>
#include <opm/autodiff/BlackoilPropsAdFromDeck.hpp>
#include <opm/autodiff/RedistributeDataHandles.hpp>
#include <opm/core/utility/share_obj.hpp>
@ -123,7 +124,7 @@ try
}
std::shared_ptr<Dune::CpGrid> grid;
std::shared_ptr<BlackoilPropertiesInterface> props;
std::shared_ptr<BlackoilPropsAdInterface> new_props;
std::shared_ptr<BlackoilPropsAdFromDeck> new_props;
std::shared_ptr<RockCompressibility> rock_comp;
BlackoilState state;
// bool check_well_controls = false;
@ -155,7 +156,7 @@ try
}
// Grid init
grid.reset(new Dune::CpGrid());
grid.reset(new Dune::CpGrid);
std::vector<double> porv = eclipseState->getDoubleGridProperty("PORV")->getData();
grid->processEclipseFormat(deck, false, false, false, porv);
@ -206,6 +207,38 @@ try
*props, deck, gravity[2], state);
}
std::shared_ptr<BlackoilState> distributed_state=Dune::stackobject_to_shared_ptr(state);
std::shared_ptr<Opm::BlackoilPropsAdFromDeck> distributed_props=new_props;
Dune::CpGrid distributed_grid = *grid;
// At this point all properties and state variables are correctly initialized
// If there are more than one processors involved, we now repartition the grid
// and initilialize new properties and states for it.
if( grid->comm().size()>=1 )
{
if(!param.getDefault("output_vtk", true))
{
OPM_THROW(std::logic_error, "We only support vtk output during parallel runs");
}
grid->loadBalance(2);
Dune::CpGrid global_grid = *grid;
distributed_grid = *grid;
global_grid.switchToGlobalView();
distributed_grid.switchToDistributedView();
distributed_props = std::make_shared<BlackoilPropsAdFromDeck>(*new_props, grid->numCells());
distributed_state.reset(new BlackoilState);
distributed_state->init(grid->numCells(), grid->numFaces(), state.numPhases());
// init does not resize surfacevol. Do it manually.
distributed_state->surfacevol().resize(grid->numCells()*state.numPhases(),
std::numeric_limits<double>::max());
Opm::BlackoilStateDataHandle state_handle(global_grid, distributed_grid,
state, *distributed_state);
Opm::BlackoilPropsDataHandle props_handle(global_grid, distributed_grid,
static_cast<BlackoilPropsAdFromDeck&>(*new_props),
static_cast<BlackoilPropsAdFromDeck&>(*distributed_props));
grid->scatterData(state_handle);
grid->scatterData(props_handle);
}
bool use_gravity = (gravity[0] != 0.0 || gravity[1] != 0.0 || gravity[2] != 0.0);
const double *grav = use_gravity ? &gravity[0] : 0;
@ -244,14 +277,14 @@ try
// initialize variables
simtimer.init(timeMap);
Opm::DerivedGeology geology(*grid, *new_props, eclipseState, false, grav);
Opm::DerivedGeology geology(distributed_grid, *distributed_props, eclipseState, false, grav);
std::vector<double> threshold_pressures = thresholdPressures(deck, eclipseState, *grid);
std::vector<double> threshold_pressures = thresholdPressures(deck, eclipseState, distributed_grid);
SimulatorFullyImplicitBlackoil<Dune::CpGrid> simulator(param,
*grid,
distributed_grid,
geology,
*new_props,
*distributed_props,
rock_comp->isActive() ? rock_comp.get() : 0,
*fis_solver,
grav,
@ -264,7 +297,7 @@ try
std::cout << "\n\n================ Starting main simulation loop ===============\n"
<< std::flush;
SimulatorReport fullReport = simulator.run(simtimer, state);
SimulatorReport fullReport = simulator.run(simtimer, *distributed_state);
std::cout << "\n\n================ End of simulation ===============\n\n";
fullReport.report(std::cout);

2
opm/autodiff/BlackoilPropsAdFromDeck.cpp
View File

@ -77,6 +77,7 @@ BlackoilPropsAdFromDeck::BlackoilPropsAdFromDeck(const BlackoilPropsAdFromDeck&
OPM_THROW(std::runtime_error, "The number of cells is has to be larger than 0.");
// Copy properties that do not depend on the postion within the grid.
rock_ = props.rock_;
satprops_ = props.satprops_;
phase_usage_ = props.phase_usage_;
props_ = props.props_;
densities_ = props.densities_;
@ -87,7 +88,6 @@ BlackoilPropsAdFromDeck::BlackoilPropsAdFromDeck(const BlackoilPropsAdFromDeck&
// and initialize with obviously bogus numbers.
cellPvtRegionIdx_.resize(number_of_cells, std::numeric_limits<int>::min());
pvtTableIdx_.resize(number_of_cells, std::numeric_limits<int>::min());
satOilMax_.resize(number_of_cells, -std::numeric_limits<double>::max());
}
/// Initializes the properties.

1
opm/autodiff/BlackoilPropsAdFromDeck.hpp
View File

@ -57,6 +57,7 @@ namespace Opm
/// version of the methods.
class BlackoilPropsAdFromDeck : public BlackoilPropsAdInterface
{
friend class BlackoilPropsDataHandle;
public:
/// Constructor wrapping an opm-core black oil interface.
BlackoilPropsAdFromDeck(Opm::DeckConstPtr deck,

3
opm/autodiff/FullyImplicitBlackoilSolver_impl.hpp
View File

@ -1906,6 +1906,7 @@ namespace {
{
const ParallelISTLInformation& info =
boost::any_cast<const ParallelISTLInformation&>(linsolver_.parallelInformation());
// Compute the global number of cells and porevolume
std::vector<int> v(nc, 1);
auto nc_and_pv = std::tuple<int, double>(0, 0.0);
@ -1913,7 +1914,7 @@ namespace {
Opm::Reduction::makeGlobalSumFunctor<double>());
auto nc_and_pv_containers = std::make_tuple(v, geo_.poreVolume());
info.computeReduction(nc_and_pv_containers, nc_and_pv_operators, nc_and_pv);
for ( int idx=0; idx<MaxNumPhases; ++idx )
{
if (active_[idx]) {

226
opm/autodiff/RedistributeDataHandles.hpp Normal file
View File

@ -0,0 +1,226 @@
/*
Copyright 2015 Dr. Blatt - HPC-Simulation-Software & Services.
Coypright 2015 NTNU
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <opm/autodiff/BlackoilPropsAdFromDeck.hpp>
#include <opm/core/simulator/BlackoilState.hpp>
namespace Opm
{
/// \brief a data handle to distribute the BlackoilState
class BlackoilStateDataHandle
{
public:
/// \brief The data that we send.
typedef double DataType;
/// \brief Constructor.
/// \param sendGrid The grid that the data is attached to when sending.
/// \param recvGrid The grid that the data is attached to when receiving.
/// \param sendState The state where we will retieve the values to be sent.
/// \parame recvState The state where we will store the received values.
BlackoilStateDataHandle(const Dune::CpGrid& sendGrid,
const Dune::CpGrid& recvGrid,
const BlackoilState& sendState,
BlackoilState& recvState)
: sendGrid_(sendGrid), recvGrid_(recvGrid), sendState_(sendState), recvState_(recvState)
{}
bool fixedsize(int /*dim*/, int /*codim*/)
{
return false;
}
template<class T>
std::size_t size(const T& e)
{
if ( T::codimension == 0)
{
return 2 * sendState_.numPhases() +4+2*sendGrid_.numCellFaces(e.index());
}
else
{
OPM_THROW(std::logic_error, "Data handle can only be used for elements");
}
}
template<class B, class T>
void gather(B& buffer, const T& e)
{
assert( T::codimension == 0);
for ( int i=0; i<sendState_.numPhases(); ++i )
{
buffer.write(sendState_.surfacevol()[e.index()*sendState_.numPhases()+i]);
}
buffer.write(sendState_.gasoilratio()[e.index()]);
buffer.write(sendState_.rv()[e.index()]);
buffer.write(sendState_.pressure()[e.index()]);
buffer.write(sendState_.temperature()[e.index()]);
buffer.write(sendState_.saturation()[e.index()]);
for ( int i=0; i<sendGrid_.numCellFaces(e.index()); ++i )
{
buffer.write(sendState_.facepressure()[sendGrid_.cellFace(e.index(), i)]);
}
for ( int i=0; i<sendGrid_.numCellFaces(e.index()); ++i )
{
buffer.write(recvState_.faceflux()[sendGrid_.cellFace(e.index(), i)]);
}
}
template<class B, class T>
void scatter(B& buffer, const T& e, std::size_t size)
{
assert( T::codimension == 0);
assert( size == 2 * recvState_.numPhases() +4+2*recvGrid_.numCellFaces(e.index()));
(void) size;
for ( int i=0; i<recvState_.numPhases(); ++i )
{
double val;
buffer.read(val);
recvState_.surfacevol()[e.index()]=val;
}
double val;
buffer.read(val);
recvState_.gasoilratio()[e.index()]=val;
buffer.read(val);
recvState_.rv()[e.index()]=val;
buffer.read(val);
recvState_.pressure()[e.index()]=val;
buffer.read(val);
recvState_.temperature()[e.index()]=val;
buffer.read(val);
recvState_.saturation()[e.index()]=val;
for ( int i=0; i<recvGrid_.numCellFaces(e.index()); ++i )
{
double val;
buffer.read(val);
recvState_.facepressure()[recvGrid_.cellFace(e.index(), i)]=val;
}
for ( int i=0; i<recvGrid_.numCellFaces(e.index()); ++i )
{
double val;
buffer.read(val);
recvState_.faceflux()[recvGrid_.cellFace(e.index(), i)]=val;
}
}
bool contains(int dim, int codim)
{
return dim==3 && codim==0;
}
private:
/// \brief The grid that the data is attached to when sending
const Dune::CpGrid& sendGrid_;
/// \brief The grid that the data is attached to when receiving
const Dune::CpGrid& recvGrid_;
/// \brief The state where we will retieve the values to be sent.
const BlackoilState& sendState_;
// \brief The state where we will store the received values.
BlackoilState& recvState_;
};
class BlackoilPropsDataHandle
{
public:
/// \brief The data that we send.
typedef double DataType;
/// \brief Constructor.
/// \param sendGrid The grid that the data is attached to when sending.
/// \param recvGrid The grid that the data is attached to when receiving.
/// \param sendProps The properties where we will retieve the values to be sent.
/// \parame recvProps The properties where we will store the received values.
BlackoilPropsDataHandle(const Dune::CpGrid& sendGrid,
const Dune::CpGrid& recvGrid,
const BlackoilPropsAdFromDeck& sendProps,
BlackoilPropsAdFromDeck& recvProps)
: sendGrid_(sendGrid), recvGrid_(recvGrid), sendProps_(sendProps), recvProps_(recvProps),
size_(2)
{
// satOilMax might be non empty. In this case we will need to send it, too.
if ( sendProps.satOilMax_.size()>0 )
{
recvProps_.satOilMax_.resize(recvGrid.numCells(),
-std::numeric_limits<double>::max());
++size_;
}
}
bool fixedsize(int /*dim*/, int /*codim*/)
{
return true;
}
template<class T>
std::size_t size(const T&)
{
if ( T::codimension == 0)
{
// We only send pvtTableIdx_, cellPvtRegionIdx_, and maybe satOilMax_
return size_;
}
else
{
OPM_THROW(std::logic_error, "Data handle can only be used for elements");
}
}
template<class B, class T>
void gather(B& buffer, const T& e)
{
assert( T::codimension == 0);
buffer.write(sendProps_.cellPvtRegionIndex()[e.index()]);
buffer.write(sendProps_.pvtTableIdx_[e.index()]);
if ( size_==2 )
return;
buffer.write(sendProps_.satOilMax_[e.index()]);
}
template<class B, class T>
void scatter(B& buffer, const T& e, std::size_t size)
{
assert( T::codimension == 0);
assert( size==size_ ); (void) size;
double val;
buffer.read(val);
recvProps_.cellPvtRegionIdx_[e.index()]=val;
buffer.read(val);
recvProps_.pvtTableIdx_[e.index()]=val;
if ( size_==2 )
return;
buffer.read(val);
recvProps_.satOilMax_[e.index()]=val;
}
bool contains(int dim, int codim)
{
return dim==3 && codim==0;
}
private:
/// \brief The grid that the data is attached to when sending
const Dune::CpGrid& sendGrid_;
/// \brief The grid that the data is attached to when receiving
const Dune::CpGrid& recvGrid_;
/// \brief The properties where we will retieve the values to be sent.
const BlackoilPropsAdFromDeck& sendProps_;
// \brief The properties where we will store the received values.
BlackoilPropsAdFromDeck& recvProps_;
std::size_t size_;
};
} // end namespace Opm