OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2406 from blattms/loadbalance-fielprops

Browse Source 
Use CpGrid::loadBalance to distribute the field properties.
This commit is contained in:
Markus Blatt 2020-03-06 11:51:14 +01:00 committed by GitHub
commit 1f00870660
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 213 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists_files.cmake
View File

@ -175,6 +175,7 @@ list (APPEND PUBLIC_HEADER_FILES
opm/simulators/utils/ParallelFileMerger.hpp
opm/simulators/utils/DeferredLoggingErrorHelpers.hpp
opm/simulators/utils/DeferredLogger.hpp
opm/simulators/utils/FieldPropsDataHandle.hpp
opm/simulators/utils/gatherDeferredLogger.hpp
opm/simulators/utils/moduleVersion.hpp
opm/simulators/utils/ParallelEclipseState.hpp

13
ebos/eclcpgridvanguard.hh
View File

@ -34,6 +34,7 @@
#include <opm/grid/CpGrid.hpp>
#include <opm/grid/cpgrid/GridHelpers.hpp>
#include <opm/simulators/utils/ParallelEclipseState.hpp>
#include <opm/simulators/utils/FieldPropsDataHandle.hpp>
#include <dune/grid/common/mcmgmapper.hh>
@ -188,7 +189,9 @@ public:
//distribute the grid and switch to the distributed view.
{
const auto wells = this->schedule().getWellsatEnd();
defunctWellNames_ = std::get<1>(grid_->loadBalance(edgeWeightsMethod, &wells, faceTrans.data()));
auto& eclState = static_cast<ParallelEclipseState&>(this->eclState());
FieldPropsDataHandle<Dune::CpGrid> handle(*grid_, eclState);
defunctWellNames_ = std::get<1>(grid_->loadBalance(handle, edgeWeightsMethod, &wells, faceTrans.data()));
}
grid_->switchToDistributedView();
@ -210,14 +213,6 @@ public:
this->updateGridView_();
#if HAVE_MPI
if (mpiSize > 1) {
std::vector<int> cartIndices;
cartIndices.reserve(grid_->numCells());
auto locElemIt = this->gridView().template begin</*codim=*/0>();
const auto& locElemEndIt = this->gridView().template end</*codim=*/0>();
for (; locElemIt != locElemEndIt; ++locElemIt) {
cartIndices.push_back(cartesianIndexMapper_->cartesianIndex(locElemIt->index()));
}
static_cast<ParallelEclipseState&>(this->eclState()).setupLocalProps(cartIndices);
static_cast<ParallelEclipseState&>(this->eclState()).switchToDistributedProps();
}
#endif

201
opm/simulators/utils/FieldPropsDataHandle.hpp Normal file
View File

@ -0,0 +1,201 @@
/*
Copyright 2020 Equinor AS.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
/*!
* \file FieldPropsDatahandle.hpp
* \brief File containing a data handle for communicating the FieldProperties
*
* \author Markus Blatt, OPM-OP AS
*/
#ifndef FIELDPROPS_DATAHANDLE_HPP
#define FIELDPROPS_DATAHANDLE_HPP
#include <opm/simulators/utils/ParallelEclipseState.hpp>
#include <opm/simulators/utils/ParallelRestart.hpp>
#include <dune/grid/common/datahandleif.hh>
#include <dune/grid/common/mcmgmapper.hh>
#include <dune/grid/common/partitionset.hh>
#include <dune/common/parallel/mpihelper.hh>
#include <unordered_map>
#include <iostream>
namespace Opm
{
/*!
* \brief A Data handle t communicate the field properties during load balance.
* \tparam Grid The type of grid where the load balancing is happening.
* \todo Maybe specialize this for CpGrid to save some space, later.
*/
template<class Grid>
class FieldPropsDataHandle
: public Dune::CommDataHandleIF< FieldPropsDataHandle<Grid>, double>
{
public:
//! \brief the data type we send (ints are converted to double)
using DataType = double;
//! \brief Constructor
//! \param grid The grid where the loadbalancing is happening.
//! \param globalProps The field properties of the global grid
//! \param distributedProps The distributed field properties
FieldPropsDataHandle(const Grid& grid, ParallelEclipseState& eclState)
: m_grid(grid), m_distributed_fieldProps(eclState.m_fieldProps)
{
// Scatter the keys
const auto& comm = Dune::MPIHelper::getCollectiveCommunication();
if (comm.rank() == 0)
{
const auto& globalProps = eclState.globalFieldProps();
m_intKeys = globalProps.keys<int>();
m_doubleKeys = globalProps.keys<double>();
std::size_t packSize = Mpi::packSize(m_intKeys, comm) +
Mpi::packSize(m_doubleKeys,comm);
std::vector<char> buffer(packSize);
int position = 0;
Mpi::pack(m_intKeys, buffer, position, comm);
Mpi::pack(m_doubleKeys, buffer, position, comm);
comm.broadcast(&position, 1, 0);
comm.broadcast(buffer.data(), position, 0);
// copy data to persistent map based on local id
auto noData = m_intKeys.size() + m_doubleKeys.size();
const auto& idSet = m_grid.localIdSet();
const auto& gridView = m_grid.levelGridView(0);
using ElementMapper =
Dune::MultipleCodimMultipleGeomTypeMapper<typename Grid::LevelGridView>;
ElementMapper elemMapper(gridView, Dune::mcmgElementLayout());
for( const auto &element : elements( gridView, Dune::Partitions::interiorBorder ) )
{
const auto& id = idSet.id(element);
auto index = elemMapper.index(element);
auto& data = elementData_[id];
data.reserve(noData);
for(const auto& intKey : m_intKeys)
data.push_back(globalProps.get_int(intKey)[index]);
for(const auto& doubleKey : m_doubleKeys)
data.push_back(globalProps.get_double(doubleKey)[index]);
}
}
else
{
int bufferSize;
comm.broadcast(&bufferSize, 1, 0);
std::vector<char> buffer(bufferSize);
comm.broadcast(buffer.data(), bufferSize, 0);
int position{};
Mpi::unpack(m_intKeys, buffer, position, comm);
Mpi::unpack(m_doubleKeys, buffer, position, comm);
}
}
FieldPropsDataHandle(const FieldPropsDataHandle&) = delete;
~FieldPropsDataHandle()
{
// distributed grid is now correctly set up.
for(const auto& intKey : m_intKeys)
m_distributed_fieldProps.m_intProps[intKey].resize(m_grid.size(0));
for(const auto& doubleKey : m_doubleKeys)
m_distributed_fieldProps.m_doubleProps[doubleKey].resize(m_grid.size(0));
// copy data for the persistent mao to the field properties
const auto& idSet = m_grid.localIdSet();
const auto& gridView = m_grid.levelGridView(0);
using ElementMapper =
Dune::MultipleCodimMultipleGeomTypeMapper<typename Grid::LevelGridView>;
ElementMapper elemMapper(gridView, Dune::mcmgElementLayout());
for( const auto &element : elements( gridView, Dune::Partitions::all ) )
{
std::size_t counter{};
const auto& id = idSet.id(element);
auto index = elemMapper.index(element);
auto data = rcvdElementData_.find(id);
assert(data != elementData_.end());
for(const auto& intKey : m_intKeys)
m_distributed_fieldProps.m_intProps[intKey][index] = static_cast<int>(data->second[counter++]);
for(const auto& doubleKey : m_doubleKeys)
m_distributed_fieldProps.m_doubleProps[doubleKey][index] = data->second[counter++];
}
}
bool contains(int /* dim */, int codim)
{
return codim == 0;
}
bool fixedsize(int /* dim */, int /* codim */)
{
return true;
}
bool fixedSize(int /* dim */, int /* codim */)
{
return true;
}
template<class EntityType>
std::size_t size(const EntityType /* entity */)
{
return m_intKeys.size() + m_doubleKeys.size();
}
template<class BufferType, class EntityType>
void gather(BufferType& buffer, const EntityType& e) const
{
auto iter = elementData_.find(m_grid.localIdSet().id(e));
assert(iter != elementData_.end());
for(const auto& data : iter->second)
buffer.write(data);
}
template<class BufferType, class EntityType>
void scatter(BufferType& buffer, const EntityType& e, std::size_t n)
{
assert(n == m_intKeys.size() + m_doubleKeys.size());
auto& array = rcvdElementData_[m_grid.localIdSet().id(e)];
array.resize(n);
for(auto& data: array)
{
buffer.read(data);
}
}
private:
using LocalIdSet = typename Grid::LocalIdSet;
const Grid& m_grid;
//! \brief The distributed field properties for receiving
ParallelFieldPropsManager& m_distributed_fieldProps;
//! \brief The names of the keys of the integer fields.
std::vector<std::string> m_intKeys;
//! \brief The names of the keys of the double fields.
std::vector<std::string> m_doubleKeys;
/// \brief The data per element as a vector mapped from the local id.
std::unordered_map<typename LocalIdSet::IdType, std::vector<double> > elementData_;
/// \brief The data per element as a vector mapped from the local id.
std::unordered_map<typename LocalIdSet::IdType, std::vector<double> > rcvdElementData_;
};
} // end namespace Opm
#endif // FIELDPROPS_DATAHANDLE_HPP

121
opm/simulators/utils/ParallelEclipseState.cpp
View File

@ -232,125 +232,4 @@ void ParallelEclipseState::switchToDistributedProps()
m_parProps = true;
}
#if HAVE_MPI
namespace {
template<class T>
struct GetField {
GetField(const FieldPropsManager& propMan) : props(propMan) {}
std::vector<T> getField(const std::string& key) const;
const FieldPropsManager& props;
};
template<>
std::vector<int> GetField<int>::getField(const std::string& key) const {
return props.get_global_int(key);
}
template<>
std::vector<double> GetField<double>::getField(const std::string& key) const {
return props.get_global_double(key);
}
template<class T>
void extractRootProps(const std::vector<int>& localToGlobal,
const std::vector<std::string>& keys,
const GetField<T>& getter,
std::map<std::string,std::vector<T>>& localMap)
{
for (const std::string& key : keys) {
auto prop = getter.getField(key);
std::vector<T>& local = localMap[key];
local.reserve(localToGlobal.size());
for (int cell : localToGlobal) {
local.push_back(prop[cell]);
}
}
}
template<class T>
void packProps(const std::vector<int>& l2gCell,
const std::vector<std::string>& keys,
const GetField<T>& getter,
std::vector<char>& buffer, int& position)
{
const auto& comm = Dune::MPIHelper::getCollectiveCommunication();
std::vector<T> sendData(l2gCell.size());
for (const std::string& key : keys) {
auto prop = getter.getField(key);
size_t idx = 0;
for (int cell : l2gCell)
sendData[idx++] = prop[cell];
Mpi::pack(sendData, buffer, position, comm);
}
}
}
void ParallelEclipseState::setupLocalProps(const std::vector<int>& localToGlobal)
{
const auto& comm = Dune::MPIHelper::getCollectiveCommunication();
if (comm.rank() == 0) {
extractRootProps(localToGlobal, this->globalFieldProps().keys<int>(),
GetField<int>(this->globalFieldProps()),
m_fieldProps.m_intProps);
extractRootProps(localToGlobal, this->globalFieldProps().keys<double>(),
GetField<double>(this->globalFieldProps()),
m_fieldProps.m_doubleProps);
for (int i = 1; i < comm.size(); ++i) {
std::vector<int> l2gCell;
size_t size;
MPI_Recv(&size, 1, Dune::MPITraits<size_t>::getType(), i, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
l2gCell.resize(size);
MPI_Recv(l2gCell.data(), size, MPI_INT, i, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
size_t cells = l2gCell.size();
const auto& intKeys = this->globalFieldProps().keys<int>();
const auto& dblKeys = this->globalFieldProps().keys<double>();
size = Mpi::packSize(intKeys, comm) +
Mpi::packSize(dblKeys,comm) +
intKeys.size() * Mpi::packSize(std::vector<int>(cells), comm) +
dblKeys.size() * Mpi::packSize(std::vector<double>(cells), comm);
std::vector<char> buffer(size);
int position = 0;
Mpi::pack(intKeys, buffer, position, comm);
Mpi::pack(dblKeys, buffer, position, comm);
packProps(l2gCell, intKeys, GetField<int>(this->globalFieldProps()),
buffer, position);
packProps(l2gCell, dblKeys, GetField<double>(this->globalFieldProps()),
buffer, position);
MPI_Send(&position, 1, MPI_INT, i, 0, MPI_COMM_WORLD);
MPI_Send(buffer.data(), position, MPI_CHAR, i, 0, MPI_COMM_WORLD);
}
} else {
size_t l2gSize = localToGlobal.size();
MPI_Send(&l2gSize, 1, Dune::MPITraits<size_t>::getType(), 0, 0, MPI_COMM_WORLD);
MPI_Send(localToGlobal.data(), localToGlobal.size(), MPI_INT, 0, 0, MPI_COMM_WORLD);
int size;
MPI_Recv(&size, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
std::vector<char> buffer(size);
MPI_Recv(buffer.data(), size, MPI_CHAR, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
std::vector<std::string> intKeys, dblKeys;
int position = 0;
Mpi::unpack(intKeys, buffer, position, comm);
Mpi::unpack(dblKeys, buffer, position, comm);
for (const std::string& key : intKeys) {
Mpi::unpack(m_fieldProps.m_intProps[key], buffer, position, comm);
}
for (const std::string& key : dblKeys) {
Mpi::unpack(m_fieldProps.m_doubleProps[key], buffer, position, comm);
}
}
}
#endif
} // end namespace Opm

14
opm/simulators/utils/ParallelEclipseState.hpp
View File

@ -20,6 +20,7 @@
#define PARALLEL_ECLIPSE_STATE_HPP
#include <opm/parser/eclipse/EclipseState/EclipseState.hpp>
//#include <opm/simulators/utils/FieldPropsDataHandle.hpp>
#include <dune/common/parallel/mpihelper.hh>
namespace Opm {
@ -38,6 +39,9 @@ class EclMpiSerializer;
class ParallelFieldPropsManager : public FieldPropsManager {
public:
friend class ParallelEclipseState; //!< Friend so props can be setup.
//! \brief Friend to set up props
template<class Grid>
friend class FieldPropsDataHandle;
//! \brief Constructor.
//! \param manager The field property manager to wrap.
@ -100,6 +104,9 @@ protected:
*/
class ParallelEclipseState : public EclipseState {
//! \brief Friend to set up props
template<class Grid>
friend class FieldPropsDataHandle;
public:
//! \brief Default constructor.
ParallelEclipseState();
@ -135,13 +142,6 @@ public:
//! setupLocalProps must be called prior to this.
void switchToDistributedProps();
#if HAVE_MPI
//! \brief Setup local properties.
//! \param localToGlobal Map from local cells on calling process to global cartesian cell
//! \details Must be called after grid has been paritioned
void setupLocalProps(const std::vector<int>& localToGlobal);
#endif
//! \brief Returns a const ref to current field properties.
const FieldPropsManager& fieldProps() const override;