From e93e18a55d730e79443ec647190d67c7f3340e97 Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Wed, 7 Sep 2022 09:39:15 +0200 Subject: [PATCH] changed: use EclMpiSerializer to distribute transmissibilities --- opm/simulators/utils/ParallelEclipseState.hpp | 12 ++++ .../utils/PropsCentroidsDataHandle.hpp | 69 ++++++++----------- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/opm/simulators/utils/ParallelEclipseState.hpp b/opm/simulators/utils/ParallelEclipseState.hpp index 94ddcd9c4..1f0666e57 100644 --- a/opm/simulators/utils/ParallelEclipseState.hpp +++ b/opm/simulators/utils/ParallelEclipseState.hpp @@ -109,6 +109,18 @@ public: void apply_tran(const std::string& keyword, std::vector& trans) const override; void deserialize_tran(const std::vector& buffer) override; + + void copyTran(const FieldPropsManager& from) + { + m_tran = from.getTran(); + } + + template + void serializeOp(Serializer& serializer) + { + serializer.map(m_tran); + } + protected: std::map> m_intProps; //!< Map of integer properties in process-local compressed indices. std::map> m_doubleProps; //!< Map of double properties in process-local compressed indices. diff --git a/opm/simulators/utils/PropsCentroidsDataHandle.hpp b/opm/simulators/utils/PropsCentroidsDataHandle.hpp index 22c0137f1..876e82ddf 100644 --- a/opm/simulators/utils/PropsCentroidsDataHandle.hpp +++ b/opm/simulators/utils/PropsCentroidsDataHandle.hpp @@ -65,44 +65,34 @@ public: const EclipseGrid* eclGridOnRoot, std::vector& centroids, const typename Dune::CartesianIndexMapper& cartMapper) - : m_grid(grid), m_distributed_fieldProps(eclState.m_fieldProps), + : m_grid(grid), + m_distributed_fieldProps(eclState.m_fieldProps), m_centroids(centroids) { // Scatter the keys const Parallel::Communication comm = m_grid.comm(); if (comm.rank() == 0) { - const auto& globalProps = eclState.globalFieldProps(); + const FieldPropsManager& globalProps = eclState.globalFieldProps(); m_intKeys = globalProps.keys(); m_doubleKeys = globalProps.keys(); - std::size_t packSize = Mpi::packSize(m_intKeys, comm) + - Mpi::packSize(m_doubleKeys,comm); - std::vector buffer(packSize); - int position = 0; - Mpi::pack(m_intKeys, buffer, position, comm); - Mpi::pack(m_doubleKeys, buffer, position, comm); - int calcStart = position; - { - std::vector tran_buffer = globalProps.serialize_tran(); - position += tran_buffer.size(); - buffer.insert(buffer.end(), std::make_move_iterator(tran_buffer.begin()), std::make_move_iterator(tran_buffer.end())); - } - comm.broadcast(&position, 1, 0); - comm.broadcast(buffer.data(), position, 0); + m_distributed_fieldProps.copyTran(globalProps); + } - // Unpack Calculator as we need it here, too. - m_distributed_fieldProps.deserialize_tran( std::vector(buffer.begin() + calcStart, buffer.end()) ); + EclMpiSerializer ser(comm); + ser.broadcast(*this); - // copy data to persistent map based on local id - m_no_data = m_intKeys.size() + m_doubleKeys.size() + - Grid::dimensionworld; + m_no_data = m_intKeys.size() + m_doubleKeys.size() + Grid::dimensionworld; + + if (comm.rank() == 0) { + const FieldPropsManager& globalProps = eclState.globalFieldProps(); const auto& idSet = m_grid.localIdSet(); const auto& gridView = m_grid.levelGridView(0); using ElementMapper = Dune::MultipleCodimMultipleGeomTypeMapper; ElementMapper elemMapper(gridView, Dune::mcmgElementLayout()); - for( const auto &element : elements( gridView, Dune::Partitions::interiorBorder ) ) + for (const auto &element : elements(gridView, Dune::Partitions::interiorBorder)) { const auto& id = idSet.id(element); auto index = elemMapper.index(element); @@ -132,31 +122,18 @@ public: data.emplace_back(center[dim], '1'); // write garbage for value_status } } - else - { - int bufferSize; - comm.broadcast(&bufferSize, 1, 0); - std::vector buffer(bufferSize); - comm.broadcast(buffer.data(), bufferSize, 0); - int position{}; - Mpi::unpack(m_intKeys, buffer, position, comm); - Mpi::unpack(m_doubleKeys, buffer, position, comm); - m_distributed_fieldProps.deserialize_tran( std::vector(buffer.begin() + position, buffer.end()) ); - m_no_data = m_intKeys.size() + m_doubleKeys.size() + - Grid::dimensionworld; - } } ~PropsCentroidsDataHandle() { // distributed grid is now correctly set up. - for(const auto& intKey : m_intKeys) + for (const auto& intKey : m_intKeys) { m_distributed_fieldProps.m_intProps[intKey].data.resize(m_grid.size(0)); m_distributed_fieldProps.m_intProps[intKey].value_status.resize(m_grid.size(0)); } - for(const auto& doubleKey : m_doubleKeys) + for (const auto& doubleKey : m_doubleKeys) { m_distributed_fieldProps.m_doubleProps[doubleKey].data.resize(m_grid.size(0)); m_distributed_fieldProps.m_doubleProps[doubleKey].value_status.resize(m_grid.size(0)); @@ -171,7 +148,7 @@ public: Dune::MultipleCodimMultipleGeomTypeMapper; ElementMapper elemMapper(gridView, Dune::mcmgElementLayout()); - for( const auto &element : elements( gridView, Dune::Partitions::all ) ) + for (const auto &element : elements( gridView, Dune::Partitions::all)) { std::size_t counter{}; const auto& id = idSet.id(element); @@ -179,14 +156,14 @@ public: auto data = elementData_.find(id); assert(data != elementData_.end()); - for(const auto& intKey : m_intKeys) + for (const auto& intKey : m_intKeys) { const auto& pair = data->second[counter++]; m_distributed_fieldProps.m_intProps[intKey].data[index] = static_cast(pair.first); m_distributed_fieldProps.m_intProps[intKey].value_status[index] = static_cast(pair.second); } - for(const auto& doubleKey : m_doubleKeys) + for (const auto& doubleKey : m_doubleKeys) { const auto& pair = data->second[counter++]; m_distributed_fieldProps.m_doubleProps[doubleKey].data[index] = pair.first; @@ -225,7 +202,7 @@ public: { auto iter = elementData_.find(m_grid.localIdSet().id(e)); assert(iter != elementData_.end()); - for(const auto& data : iter->second) + for (const auto& data : iter->second) { buffer.write(data); } @@ -237,12 +214,20 @@ public: assert(n == m_no_data); auto& array = elementData_[m_grid.localIdSet().id(e)]; array.resize(n); - for(auto& data: array) + for (auto& data : array) { buffer.read(data); } } + template + void serializeOp(Serializer& serializer) + { + serializer(m_intKeys); + serializer(m_doubleKeys); + m_distributed_fieldProps.serializeOp(serializer); + } + private: using LocalIdSet = typename Grid::LocalIdSet; const Grid& m_grid;