From e8ebf0840b25d426a14f5613aec02bc38f5e6066 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Tue, 19 May 2015 16:13:59 +0200 Subject: [PATCH 1/5] Allow to create the correct communication information if there are several unknowns. In this case the parallel index set might represent N entries (this might be the number of cells of grid). Nevertheless, there several (n) equations/unknowns attached to each index. In this case we construct a larger index set representing N*n unknows, where each unknown is attached to an index. This change only affects parallel runs. --- opm/core/linalg/ParallelIstlInformation.hpp | 39 ++++++++++++++++----- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/opm/core/linalg/ParallelIstlInformation.hpp b/opm/core/linalg/ParallelIstlInformation.hpp index ce866383..cc33446b 100644 --- a/opm/core/linalg/ParallelIstlInformation.hpp +++ b/opm/core/linalg/ParallelIstlInformation.hpp @@ -28,7 +28,6 @@ #include #if HAVE_MPI && HAVE_DUNE_ISTL - #include "mpi.h" #include #include @@ -38,6 +37,7 @@ #include #include #include +#include #include namespace Opm @@ -112,10 +112,21 @@ public: /// \brief Copy the information stored to the specified objects. /// \param[out] indexSet The object to store the index set in. /// \param[out] remoteIndices The object to store the remote indices information in. - void copyValuesTo(ParallelIndexSet& indexSet, RemoteIndices& remoteIndices) const + void copyValuesTo(ParallelIndexSet& indexSet, RemoteIndices& remoteIndices, + std::size_t local_component_size=0, std::size_t no_components=1) const { + ParallelIndexSet::GlobalIndex max_gi = local_component_size; + if (no_components>1) + { + // component the max global index + for( auto i = indexSet_->begin(), end = indexSet_->end(); i != end; ++i ) + max_gi = std::max(max_gi, i->global()); + ++max_gi; + max_gi = communicator_.max(max_gi); + } indexSet.beginResize(); - IndexSetInserter inserter(indexSet); + IndexSetInserter inserter(indexSet, max_gi, + local_component_size, no_components); std::for_each(indexSet_->begin(), indexSet_->end(), inserter); indexSet.endResize(); remoteIndices.rebuild(); @@ -317,19 +328,31 @@ private: { public: typedef T ParallelIndexSet; + typedef typename ParallelIndexSet::LocalIndex LocalIndex; + typedef typename ParallelIndexSet::GlobalIndex GlobalIndex; - IndexSetInserter(ParallelIndexSet& indexSet) - : indexSet_(&indexSet) + IndexSetInserter(ParallelIndexSet& indexSet, const GlobalIndex& component_size, + std::size_t local_component_size, std::size_t no_components) + : indexSet_(&indexSet), component_size_(component_size), + local_component_size_(local_component_size), + no_components_(no_components) {} void operator()(const typename ParallelIndexSet::IndexPair& pair) { - indexSet_->add(pair.global(), pair.local()); + for(std::size_t i = 0; i < no_components_; i++) + indexSet_->add(i * component_size_ + pair.global(), + LocalIndex(i * local_component_size_ + pair.local(), + pair.local().attribute())); } - private: ParallelIndexSet* indexSet_; + /// \brief The global number of unknowns per component/equation. + GlobalIndex component_size_; + /// \brief The local number of unknowns per component/equation. + std::size_t local_component_size_; + /// \brief The number of components/equations. + std::size_t no_components_; }; - std::shared_ptr indexSet_; std::shared_ptr remoteIndices_; Dune::CollectiveCommunication communicator_; From 3b359df5343534e0cc9ee9b4bafac833c5c60218 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Tue, 19 May 2015 16:20:57 +0200 Subject: [PATCH 2/5] Update copyright information. --- opm/core/linalg/ParallelIstlInformation.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opm/core/linalg/ParallelIstlInformation.hpp b/opm/core/linalg/ParallelIstlInformation.hpp index cc33446b..90e086ca 100644 --- a/opm/core/linalg/ParallelIstlInformation.hpp +++ b/opm/core/linalg/ParallelIstlInformation.hpp @@ -1,6 +1,6 @@ /* Copyright 2014, 2015 Dr. Markus Blatt - HPC-Simulation-Software & Services - Copyright 2014 Statoil ASA + Copyright 2014, 2015 Statoil ASA Copyright 2015 NTNU This file is part of the Open Porous Media project (OPM). From 40fec324d52be392e941cefde293741f3001923a Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Tue, 19 May 2015 19:53:37 +0200 Subject: [PATCH 3/5] Rename no_components to num_components. --- opm/core/linalg/ParallelIstlInformation.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/opm/core/linalg/ParallelIstlInformation.hpp b/opm/core/linalg/ParallelIstlInformation.hpp index 90e086ca..95abc46c 100644 --- a/opm/core/linalg/ParallelIstlInformation.hpp +++ b/opm/core/linalg/ParallelIstlInformation.hpp @@ -113,10 +113,10 @@ public: /// \param[out] indexSet The object to store the index set in. /// \param[out] remoteIndices The object to store the remote indices information in. void copyValuesTo(ParallelIndexSet& indexSet, RemoteIndices& remoteIndices, - std::size_t local_component_size=0, std::size_t no_components=1) const + std::size_t local_component_size=0, std::size_t num_components=1) const { ParallelIndexSet::GlobalIndex max_gi = local_component_size; - if (no_components>1) + if ( num_components > 1 ) { // component the max global index for( auto i = indexSet_->begin(), end = indexSet_->end(); i != end; ++i ) @@ -126,7 +126,7 @@ public: } indexSet.beginResize(); IndexSetInserter inserter(indexSet, max_gi, - local_component_size, no_components); + local_component_size, num_components); std::for_each(indexSet_->begin(), indexSet_->end(), inserter); indexSet.endResize(); remoteIndices.rebuild(); @@ -332,14 +332,14 @@ private: typedef typename ParallelIndexSet::GlobalIndex GlobalIndex; IndexSetInserter(ParallelIndexSet& indexSet, const GlobalIndex& component_size, - std::size_t local_component_size, std::size_t no_components) + std::size_t local_component_size, std::size_t num_components) : indexSet_(&indexSet), component_size_(component_size), local_component_size_(local_component_size), - no_components_(no_components) + num_components_(num_components) {} void operator()(const typename ParallelIndexSet::IndexPair& pair) { - for(std::size_t i = 0; i < no_components_; i++) + for(std::size_t i = 0; i < num_components_; i++) indexSet_->add(i * component_size_ + pair.global(), LocalIndex(i * local_component_size_ + pair.local(), pair.local().attribute())); @@ -351,7 +351,7 @@ private: /// \brief The local number of unknowns per component/equation. std::size_t local_component_size_; /// \brief The number of components/equations. - std::size_t no_components_; + std::size_t num_components_; }; std::shared_ptr indexSet_; std::shared_ptr remoteIndices_; From f0a71e8898a68c6777b2d3bad290f636d6ac8e16 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Tue, 19 May 2015 19:56:37 +0200 Subject: [PATCH 4/5] Add spaces around binaries and explicit braces. --- opm/core/linalg/ParallelIstlInformation.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opm/core/linalg/ParallelIstlInformation.hpp b/opm/core/linalg/ParallelIstlInformation.hpp index 95abc46c..9f2c2200 100644 --- a/opm/core/linalg/ParallelIstlInformation.hpp +++ b/opm/core/linalg/ParallelIstlInformation.hpp @@ -113,14 +113,16 @@ public: /// \param[out] indexSet The object to store the index set in. /// \param[out] remoteIndices The object to store the remote indices information in. void copyValuesTo(ParallelIndexSet& indexSet, RemoteIndices& remoteIndices, - std::size_t local_component_size=0, std::size_t num_components=1) const + std::size_t local_component_size = 0, std::size_t num_components = 1) const { ParallelIndexSet::GlobalIndex max_gi = local_component_size; if ( num_components > 1 ) { // component the max global index for( auto i = indexSet_->begin(), end = indexSet_->end(); i != end; ++i ) + { max_gi = std::max(max_gi, i->global()); + } ++max_gi; max_gi = communicator_.max(max_gi); } From 9da18a14e13f5ba9cb63053fabb4a55fb90d7f37 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Tue, 19 May 2015 19:58:05 +0200 Subject: [PATCH 5/5] Use more accurate name for the size of the global components. The new name is num_global_components, which actually is an upper bound for the the number of global components (1 plus the maximum global index). --- opm/core/linalg/ParallelIstlInformation.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/opm/core/linalg/ParallelIstlInformation.hpp b/opm/core/linalg/ParallelIstlInformation.hpp index 9f2c2200..24e9084d 100644 --- a/opm/core/linalg/ParallelIstlInformation.hpp +++ b/opm/core/linalg/ParallelIstlInformation.hpp @@ -115,19 +115,20 @@ public: void copyValuesTo(ParallelIndexSet& indexSet, RemoteIndices& remoteIndices, std::size_t local_component_size = 0, std::size_t num_components = 1) const { - ParallelIndexSet::GlobalIndex max_gi = local_component_size; + ParallelIndexSet::GlobalIndex global_component_size = local_component_size; if ( num_components > 1 ) { + ParallelIndexSet::GlobalIndex max_gi = 0; // component the max global index for( auto i = indexSet_->begin(), end = indexSet_->end(); i != end; ++i ) { max_gi = std::max(max_gi, i->global()); } - ++max_gi; - max_gi = communicator_.max(max_gi); + global_component_size = max_gi+1; + global_component_size = communicator_.max(global_component_size); } indexSet.beginResize(); - IndexSetInserter inserter(indexSet, max_gi, + IndexSetInserter inserter(indexSet, global_component_size, local_component_size, num_components); std::for_each(indexSet_->begin(), indexSet_->end(), inserter); indexSet.endResize();