From 653cb28bc2e686fb22eb59fc0685c91be43e9581 Mon Sep 17 00:00:00 2001 From: Lisa Julia Nebel Date: Thu, 11 Jul 2024 14:25:04 +0200 Subject: [PATCH 1/2] Add the metis partitioner as a command line option as well --- CMakeLists_files.cmake | 2 +- opm/simulators/flow/CpGridVanguard.hpp | 13 ++-- opm/simulators/flow/FlowBaseVanguard.hpp | 53 ++++++++++++++- opm/simulators/flow/FlowGenericVanguard.hpp | 27 +++++++- opm/simulators/flow/GenericCpGridVanguard.cpp | 33 +++++----- opm/simulators/flow/GenericCpGridVanguard.hpp | 10 ++- opm/simulators/flow/partitionCells.cpp | 2 +- ...Params.cpp => SetupPartitioningParams.cpp} | 64 ++++++++++++++----- ...Params.hpp => SetupPartitioningParams.hpp} | 7 +- 9 files changed, 164 insertions(+), 47 deletions(-) rename opm/simulators/utils/{SetupZoltanParams.cpp => SetupPartitioningParams.cpp} (54%) rename opm/simulators/utils/{SetupZoltanParams.hpp => SetupPartitioningParams.hpp} (81%) diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index d50c32860..75dc943ab 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -287,7 +287,7 @@ if(MPI_FOUND) opm/simulators/utils/ParallelEclipseState.cpp opm/simulators/utils/ParallelNLDDPartitioningZoltan.cpp opm/simulators/utils/ParallelSerialization.cpp - opm/simulators/utils/SetupZoltanParams.cpp) + opm/simulators/utils/SetupPartitioningParams.cpp) list(APPEND PUBLIC_HEADER_FILES opm/simulators/utils/MPIPacker.hpp opm/simulators/utils/MPISerializer.hpp) endif() diff --git a/opm/simulators/flow/CpGridVanguard.hpp b/opm/simulators/flow/CpGridVanguard.hpp index 04cccfce7..faca2b0ac 100644 --- a/opm/simulators/flow/CpGridVanguard.hpp +++ b/opm/simulators/flow/CpGridVanguard.hpp @@ -227,10 +227,11 @@ public: } this->doLoadBalance_(this->edgeWeightsMethod(), this->ownersFirst(), - this->serialPartitioning(), this->enableDistributedWells(), - this->zoltanImbalanceTol(), this->gridView(), - this->schedule(), this->eclState(), - this->parallelWells_, this->numJacobiBlocks()); + this->partitionMethod(), this->serialPartitioning(), + this->enableDistributedWells(), this->imbalanceTol(), + this->gridView(), this->schedule(), + this->eclState(), this->parallelWells_, + this->numJacobiBlocks()); #endif this->updateGridView_(); @@ -298,6 +299,10 @@ protected: { return this->zoltanParams_; } + const std::string& metisParams() const override + { + return this->metisParams_; + } #endif // removing some connection located in inactive grid cells diff --git a/opm/simulators/flow/FlowBaseVanguard.hpp b/opm/simulators/flow/FlowBaseVanguard.hpp index 418464bd8..ebd6df4f8 100644 --- a/opm/simulators/flow/FlowBaseVanguard.hpp +++ b/opm/simulators/flow/FlowBaseVanguard.hpp @@ -105,11 +105,22 @@ struct OwnerCellsFirst { using type = UndefinedProperty; }; +template +struct PartitionMethod { + using type = UndefinedProperty; +}; + template struct SerialPartitioning { using type = UndefinedProperty; }; +template +struct ImbalanceTol { + using type = UndefinedProperty; +}; + +// Remove this for release 2025.04 template struct ZoltanImbalanceTol { using type = UndefinedProperty; @@ -120,6 +131,11 @@ struct ZoltanParams { using type = UndefinedProperty; }; +template +struct MetisParams { + using type = UndefinedProperty; +}; + template struct ExternalPartition { @@ -175,6 +191,12 @@ template struct OwnerCellsFirst { static constexpr bool value = true; }; + +template +struct PartitionMethod { + static constexpr int value = 1; // 0: simple, 1: Zoltan, 2: METIS, see GridEnums.hpp +}; + template struct SerialPartitioning { static constexpr bool value = false; @@ -190,6 +212,16 @@ struct ZoltanParams { static constexpr auto value = "graph"; }; +template +struct ImbalanceTol { + static constexpr double value = 1.1; +}; + +template +struct MetisParams { + static constexpr auto value = "default"; +}; + template struct ExternalPartition { @@ -280,10 +312,13 @@ public: Parameters::registerParam ("Order cells owned by rank before ghost/overlap cells."); #if HAVE_MPI + Parameters::registerParam + ("Choose partitioning strategy: 0=simple, 1=Zoltan, 2=METIS."); Parameters::registerParam ("Perform partitioning for parallel runs on a single process."); Parameters::registerParam - ("Tolerable imbalance of the loadbalancing provided by Zoltan (default: 1.1)."); + ("Tolerable imbalance of the loadbalancing provided by Zoltan. DEPRECATED: Use --imbalance-tol instead"); + Parameters::hideParam(); Parameters::registerParam ("Configuration of Zoltan partitioner. " "Valid options are: graph, hypergraph or scotch. " @@ -292,6 +327,15 @@ public: "See https://sandialabs.github.io/Zoltan/ug_html/ug.html " "for available Zoltan options."); Parameters::hideParam(); + Parameters::registerParam + ("Tolerable imbalance of the loadbalancing (default: 1.1)."); + + Parameters::registerParam + ("Configuration of Metis partitioner. " + "You can request a configuration to be read " + "from a JSON file by giving the filename here, ending with '.json.' " + "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf" + "for available METIS options."); Parameters::registerParam ("Name of file from which to load an externally generated " "partitioning of the model's active cells for MPI " @@ -324,9 +368,16 @@ public: ownersFirst_ = Parameters::get(); #if HAVE_MPI + partitionMethod_ = Dune::PartitionMethod(Parameters::get()); serialPartitioning_ = Parameters::get(); + imbalanceTol_ = Parameters::get(); + + zoltanImbalanceTolSet_ = Parameters::isSet(); zoltanImbalanceTol_ = Parameters::get(); zoltanParams_ = Parameters::get(); + + metisParams_ = Parameters::get(); + externalPartitionFile_ = Parameters::get(); #endif enableDistributedWells_ = Parameters::get(); diff --git a/opm/simulators/flow/FlowGenericVanguard.hpp b/opm/simulators/flow/FlowGenericVanguard.hpp index 48ed97d59..9d6298876 100644 --- a/opm/simulators/flow/FlowGenericVanguard.hpp +++ b/opm/simulators/flow/FlowGenericVanguard.hpp @@ -29,6 +29,8 @@ #include +#include + #include #include @@ -213,6 +215,11 @@ public: { return ownersFirst_; } #if HAVE_MPI + /*! + * \brief Parameter deciding which partition method to use + */ + Dune::PartitionMethod partitionMethod() const + { return partitionMethod_; } /*! * \brief Parameter that decides if partitioning for parallel runs * should be performed on a single process only. @@ -221,10 +228,17 @@ public: { return serialPartitioning_; } /*! - * \brief Parameter that sets the zoltan imbalance tolarance. + * \brief Parameter that sets the imbalance tolarance, depending on the chosen partition method */ - double zoltanImbalanceTol() const - { return zoltanImbalanceTol_; } + double imbalanceTol() const + { + if (zoltanImbalanceTolSet_) { + OpmLog::info("The parameter --zoltan-imbalance-tol is deprecated and has been renamed to --imbalance-tol, please adjust your calls and scripts!"); + return zoltanImbalanceTol_; + } else { + return imbalanceTol_; + } + } const std::string& externalPartitionFile() const { @@ -291,9 +305,16 @@ protected: bool ownersFirst_; #if HAVE_MPI + Dune::PartitionMethod partitionMethod_; bool serialPartitioning_; + double imbalanceTol_; + + bool zoltanImbalanceTolSet_; double zoltanImbalanceTol_; std::string zoltanParams_; + + std::string metisParams_; + std::string externalPartitionFile_{}; #endif bool enableDistributedWells_; diff --git a/opm/simulators/flow/GenericCpGridVanguard.cpp b/opm/simulators/flow/GenericCpGridVanguard.cpp index 68ccd1b22..97766179a 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.cpp +++ b/opm/simulators/flow/GenericCpGridVanguard.cpp @@ -40,7 +40,7 @@ #include #include #include -#include +#include #if HAVE_MPI #include @@ -146,17 +146,20 @@ template void GenericCpGridVanguard:: doLoadBalance_(const Dune::EdgeWeightMethod edgeWeightsMethod, const bool ownersFirst, + const Dune::PartitionMethod partitionMethod, const bool serialPartitioning, const bool enableDistributedWells, - const double zoltanImbalanceTol, + const double imbalanceTol, const GridView& gridView, const Schedule& schedule, EclipseState& eclState1, FlowGenericVanguard::ParallelWellStruct& parallelWells, const int numJacobiBlocks) { - if (!this->zoltanParams().empty()) - this->grid_->setZoltanParams(setupZoltanParams(this->zoltanParams())); + if (partitionMethod == Dune::PartitionMethod::zoltan && !this->zoltanParams().empty()) + this->grid_->setPartitioningParams(setupZoltanParams(this->zoltanParams())); + if (partitionMethod == Dune::PartitionMethod::metis && !this->metisParams().empty()) + this->grid_->setPartitioningParams(setupMetisParams(this->metisParams())); const auto mpiSize = this->grid_->comm().size(); @@ -196,9 +199,9 @@ doLoadBalance_(const Dune::EdgeWeightMethod edgeWeightsMethod, // Distribute the grid and switch to the distributed view. if (mpiSize > 1) { - this->distributeGrid(edgeWeightsMethod, ownersFirst, + this->distributeGrid(edgeWeightsMethod, ownersFirst, partitionMethod, serialPartitioning, enableDistributedWells, - zoltanImbalanceTol, loadBalancerSet != 0, + imbalanceTol, loadBalancerSet != 0, faceTrans, wells, eclState1, parallelWells); } @@ -214,7 +217,7 @@ doLoadBalance_(const Dune::EdgeWeightMethod edgeWeightsMethod, this->cell_part_ = this->grid_-> zoltanPartitionWithoutScatter(&wells, faceTrans.data(), numJacobiBlocks, - zoltanImbalanceTol); + imbalanceTol); } #endif } @@ -281,9 +284,10 @@ void GenericCpGridVanguard:: distributeGrid(const Dune::EdgeWeightMethod edgeWeightsMethod, const bool ownersFirst, + const Dune::PartitionMethod partitionMethod, const bool serialPartitioning, const bool enableDistributedWells, - const double zoltanImbalanceTol, + const double imbalanceTol, const bool loadBalancerSet, const std::vector& faceTrans, const std::vector& wells, @@ -293,9 +297,9 @@ distributeGrid(const Dune::EdgeWeightMethod edgeWeightsMethod, if (auto* eclState = dynamic_cast(&eclState1); eclState != nullptr) { - this->distributeGrid(edgeWeightsMethod, ownersFirst, + this->distributeGrid(edgeWeightsMethod, ownersFirst, partitionMethod, serialPartitioning, enableDistributedWells, - zoltanImbalanceTol, loadBalancerSet, faceTrans, + imbalanceTol, loadBalancerSet, faceTrans, wells, eclState, parallelWells); } else { @@ -317,9 +321,10 @@ void GenericCpGridVanguard:: distributeGrid(const Dune::EdgeWeightMethod edgeWeightsMethod, const bool ownersFirst, + const Dune::PartitionMethod partitionMethod, const bool serialPartitioning, const bool enableDistributedWells, - const double zoltanImbalanceTol, + const double imbalanceTol, const bool loadBalancerSet, const std::vector& faceTrans, const std::vector& wells, @@ -340,20 +345,18 @@ distributeGrid(const Dune::EdgeWeightMethod edgeWeightsMethod, auto parts = isIORank ? (*externalLoadBalancer)(*this->grid_) : std::vector{}; - + //For this case, simple partitioning is selected automatically parallelWells = std::get<1>(this->grid_->loadBalance(handle, parts, &wells, ownersFirst, addCornerCells, overlapLayers)); } else { - const auto useZoltan = true; - parallelWells = std::get<1>(this->grid_->loadBalance(handle, edgeWeightsMethod, &wells, serialPartitioning, faceTrans.data(), ownersFirst, addCornerCells, overlapLayers, - useZoltan, zoltanImbalanceTol, + partitionMethod, imbalanceTol, enableDistributedWells)); } } diff --git a/opm/simulators/flow/GenericCpGridVanguard.hpp b/opm/simulators/flow/GenericCpGridVanguard.hpp index 76db2eaf9..256da4191 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.hpp +++ b/opm/simulators/flow/GenericCpGridVanguard.hpp @@ -150,9 +150,10 @@ protected: #if HAVE_MPI void doLoadBalance_(const Dune::EdgeWeightMethod edgeWeightsMethod, const bool ownersFirst, + const Dune::PartitionMethod partitionMethod, const bool serialPartitioning, const bool enableDistributedWells, - const double zoltanImbalanceTol, + const double imbalanceTol, const GridView& gridView, const Schedule& schedule, EclipseState& eclState, @@ -166,9 +167,10 @@ private: void distributeGrid(const Dune::EdgeWeightMethod edgeWeightsMethod, const bool ownersFirst, + const Dune::PartitionMethod partitionMethod, const bool serialPartitioning, const bool enableDistributedWells, - const double zoltanImbalanceTol, + const double imbalanceTol, const bool loadBalancerSet, const std::vector& faceTrans, const std::vector& wells, @@ -177,9 +179,10 @@ private: void distributeGrid(const Dune::EdgeWeightMethod edgeWeightsMethod, const bool ownersFirst, + const Dune::PartitionMethod partitionMethod, const bool serialPartitioning, const bool enableDistributedWells, - const double zoltanImbalanceTol, + const double imbalanceTol, const bool loadBalancerSet, const std::vector& faceTrans, const std::vector& wells, @@ -188,6 +191,7 @@ private: protected: virtual const std::string& zoltanParams() const = 0; + virtual const std::string& metisParams() const = 0; #endif // HAVE_MPI diff --git a/opm/simulators/flow/partitionCells.cpp b/opm/simulators/flow/partitionCells.cpp index c148f7eef..2ade0aeba 100644 --- a/opm/simulators/flow/partitionCells.cpp +++ b/opm/simulators/flow/partitionCells.cpp @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include diff --git a/opm/simulators/utils/SetupZoltanParams.cpp b/opm/simulators/utils/SetupPartitioningParams.cpp similarity index 54% rename from opm/simulators/utils/SetupZoltanParams.cpp rename to opm/simulators/utils/SetupPartitioningParams.cpp index bbee2bad1..aaa8e8b2c 100644 --- a/opm/simulators/utils/SetupZoltanParams.cpp +++ b/opm/simulators/utils/SetupPartitioningParams.cpp @@ -18,19 +18,42 @@ */ #include -#include +#include #include #include +#if BOOST_VERSION / 100 % 1000 > 48 #include #include +#endif #include +#include namespace Opm +{ +#if BOOST_VERSION / 100 % 1000 > 48 +void convertJSONToMap(const std::string& conf, std::map& result) { + if ( !std::filesystem::exists(conf) ) { + OPM_THROW(std::invalid_argument, + "JSON file " + conf + " does not exist."); + } + boost::property_tree::ptree tree; + try { + boost::property_tree::read_json(conf, tree); + } catch (boost::property_tree::json_parser::json_parser_error& err) { + OpmLog::error(err.what()); + } + for (const auto& node : tree) { + auto value = node.second.get_value_optional(); + if (value) + result.insert_or_assign(node.first, *value); + } +} +#endif std::map setupZoltanParams(const std::string& conf) { @@ -46,21 +69,7 @@ std::map setupZoltanParams(const std::string& conf) result.emplace("GRAPH_PACKAGE", "PHG"); } else if (conf.size() > 5 && conf.substr(conf.size() - 5, 5) == ".json") { #if BOOST_VERSION / 100 % 1000 > 48 - if ( !std::filesystem::exists(conf) ) { - OPM_THROW(std::invalid_argument, - "JSON file " + conf + " does not exist."); - } - boost::property_tree::ptree tree; - try { - boost::property_tree::read_json(conf, tree); - } catch (boost::property_tree::json_parser::json_parser_error& err) { - OpmLog::error(err.what()); - } - for (const auto& node : tree) { - auto value = node.second.get_value_optional(); - if (value) - result.insert_or_assign(node.first, *value); - } + convertJSONToMap(conf, result); #else OPM_THROW(std::invalid_argument, "--zoltan-params=file.json not supported with " @@ -76,4 +85,27 @@ std::map setupZoltanParams(const std::string& conf) return result; } +std::map setupMetisParams(const std::string& conf) +{ + std::map result; + if (conf == "default") { + return result; + } else if (conf.size() > 5 && conf.substr(conf.size() - 5, 5) == ".json") { +#if BOOST_VERSION / 100 % 1000 > 48 + convertJSONToMap(conf, result); + return result; +#else + OPM_THROW(std::invalid_argument, + "--metis-params=file.json not supported with " + "boost version. Needs version > 1.48."); +#endif + } else { + // No valid configuration option found. + OPM_THROW(std::invalid_argument, + conf + " is not a valid setting for --metis-params." + " Please use a json file containing the METIS parameters."); + } +} + + } // namespace Opm diff --git a/opm/simulators/utils/SetupZoltanParams.hpp b/opm/simulators/utils/SetupPartitioningParams.hpp similarity index 81% rename from opm/simulators/utils/SetupZoltanParams.hpp rename to opm/simulators/utils/SetupPartitioningParams.hpp index 1e77c3a78..b53be695f 100644 --- a/opm/simulators/utils/SetupZoltanParams.hpp +++ b/opm/simulators/utils/SetupPartitioningParams.hpp @@ -17,8 +17,8 @@ along with OPM. If not, see . */ -#ifndef OPM_SETUP_ZOLTAN_PARAMS_HPP -#define OPM_SETUP_ZOLTAN_PARAMS_HPP +#ifndef OPM_SETUP_PARTITIONING_PARAMS_HPP +#define OPM_SETUP_PARTITIONING_PARAMS_HPP #include #include @@ -26,7 +26,8 @@ namespace Opm { std::map setupZoltanParams(const std::string& conf); +std::map setupMetisParams(const std::string& conf); } // namespace Opm -#endif // OPM_SETUP_ZOLTAN_PARAMS_HPP +#endif // OPM_SETUP_PARTITIONING_PARAMS_HPP From 371fbdfc0313093f0c8275eb1950c6c5eb5c3a06 Mon Sep 17 00:00:00 2001 From: Lisa Julia Nebel Date: Tue, 16 Jul 2024 16:17:31 +0200 Subject: [PATCH 2/2] Include before the macro BOOST_VERSION is checked --- opm/simulators/utils/SetupPartitioningParams.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opm/simulators/utils/SetupPartitioningParams.cpp b/opm/simulators/utils/SetupPartitioningParams.cpp index aaa8e8b2c..92349e4de 100644 --- a/opm/simulators/utils/SetupPartitioningParams.cpp +++ b/opm/simulators/utils/SetupPartitioningParams.cpp @@ -23,9 +23,10 @@ #include #include +#include + #if BOOST_VERSION / 100 % 1000 > 48 #include -#include #endif #include