From f16080eb210c42a41abcb7b5fea4a7484ec1424e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A5rd=20Skaflestad?= Date: Tue, 25 Feb 2025 16:52:49 +0100 Subject: [PATCH 1/2] Revise Parameter Descriptions for Partitioning Parameters Slight rewording and splitting long lines for the most part. --- opm/simulators/flow/FlowGenericVanguard.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/opm/simulators/flow/FlowGenericVanguard.cpp b/opm/simulators/flow/FlowGenericVanguard.cpp index 4762ee46c2..2190fc5f5a 100644 --- a/opm/simulators/flow/FlowGenericVanguard.cpp +++ b/opm/simulators/flow/FlowGenericVanguard.cpp @@ -468,21 +468,22 @@ void FlowGenericVanguard::registerParameters_() Parameters::Register ("Perform partitioning for parallel runs on a single process."); Parameters::Register> - ("Tolerable imbalance of the loadbalancing provided by Zoltan. DEPRECATED: Use --imbalance-tol instead"); + ("Tolerable imbalance of the loadbalancing provided by Zoltan. " + "DEPRECATED: Use --imbalance-tol instead."); Parameters::Register ("Configuration of Zoltan partitioner. " "Valid options are: graph, hypergraph or scotch. " "Alternatively, you can request a configuration to be read " - "from a JSON file by giving the filename here, ending with '.json.' " + "from a JSON file by giving the filename here, with extension '.json'. " "See https://sandialabs.github.io/Zoltan/ug_html/ug.html " "for available Zoltan options."); Parameters::Register> - ("Tolerable imbalance of the loadbalancing (default: 1.1)."); + ("Tolerable imbalance of the loadbalancing."); Parameters::Register ("Configuration of Metis partitioner. " "You can request a configuration to be read " - "from a JSON file by giving the filename here, ending with '.json.' " - "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf" + "from a JSON file by giving the filename here, with extension '.json'. " + "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf " "for available METIS options."); Parameters::Register ("Name of file from which to load an externally generated " From fed4a3bb1c5528d20fa8e32b059d0b1de39f7633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A5rd=20Skaflestad?= Date: Tue, 25 Feb 2025 16:56:24 +0100 Subject: [PATCH 2/2] Make Zoltan's Hyperedge Size Threshold Runtime Controllable This commit introduces a new hidden runtime parameter, ZoltanPhgEdgeSizeThreshold (--zoltan-phg-edge-size-threshold, double, default value = 0.35) which maps to Zoltan's low-level control parameter PHG_EDGE_SIZE_THRESHOLD that controls which hypergraph edges to omit/discard. We add a new parameter to 'setupZoltanParameters()' and thread the command line parameter through as an argument to this function. This is to enable runtime experimentation with this aspect of the partitioning algorithm, but end-users should typically not alter the default value without good reason. --- opm/simulators/flow/CpGridVanguard.hpp | 6 +++ opm/simulators/flow/FlowGenericVanguard.cpp | 7 +++- opm/simulators/flow/FlowGenericVanguard.hpp | 3 ++ opm/simulators/flow/GenericCpGridVanguard.cpp | 19 ++++++--- opm/simulators/flow/GenericCpGridVanguard.hpp | 1 + .../utils/SetupPartitioningParams.cpp | 32 +++++++++++---- .../utils/SetupPartitioningParams.hpp | 39 ++++++++++++++++++- 7 files changed, 91 insertions(+), 16 deletions(-) diff --git a/opm/simulators/flow/CpGridVanguard.hpp b/opm/simulators/flow/CpGridVanguard.hpp index 28da073618..22beeaca18 100644 --- a/opm/simulators/flow/CpGridVanguard.hpp +++ b/opm/simulators/flow/CpGridVanguard.hpp @@ -316,6 +316,12 @@ protected: { return this->zoltanParams_; } + + double zoltanPhgEdgeSizeThreshold() const override + { + return this->zoltanPhgEdgeSizeThreshold_; + } + const std::string& metisParams() const override { return this->metisParams_; diff --git a/opm/simulators/flow/FlowGenericVanguard.cpp b/opm/simulators/flow/FlowGenericVanguard.cpp index 2190fc5f5a..b0da78f649 100644 --- a/opm/simulators/flow/FlowGenericVanguard.cpp +++ b/opm/simulators/flow/FlowGenericVanguard.cpp @@ -124,7 +124,7 @@ FlowGenericVanguard::FlowGenericVanguard(SimulationModelParams&& params) partitionMethod_ = Dune::PartitionMethod(Parameters::Get()); serialPartitioning_ = Parameters::Get(); zoltanParams_ = Parameters::Get(); - + zoltanPhgEdgeSizeThreshold_ = Parameters::Get(); metisParams_ = Parameters::Get(); externalPartitionFile_ = Parameters::Get(); @@ -479,6 +479,10 @@ void FlowGenericVanguard::registerParameters_() "for available Zoltan options."); Parameters::Register> ("Tolerable imbalance of the loadbalancing."); + Parameters::Register + ("Low-level threshold fraction in the range [0,1] controlling " + "which hypergraph edge to omit. Used if --zoltan-params=\"graph\" " + "or if --zoltan-params=\"hypergraph\"."); Parameters::Register ("Configuration of Metis partitioner. " "You can request a configuration to be read " @@ -494,6 +498,7 @@ void FlowGenericVanguard::registerParameters_() Parameters::Hide>(); Parameters::Hide(); + Parameters::Hide(); #endif // HAVE_MPI Parameters::Register diff --git a/opm/simulators/flow/FlowGenericVanguard.hpp b/opm/simulators/flow/FlowGenericVanguard.hpp index 96012d3532..b7a462c462 100644 --- a/opm/simulators/flow/FlowGenericVanguard.hpp +++ b/opm/simulators/flow/FlowGenericVanguard.hpp @@ -84,6 +84,8 @@ struct SerialPartitioning{ static constexpr bool value = false; }; template struct ZoltanImbalanceTol { static constexpr Scalar value = 1.1; }; +struct ZoltanPhgEdgeSizeThreshold { static constexpr auto value = 0.35; }; + struct ZoltanParams { static constexpr auto value = "graph"; }; } // namespace Opm::Parameters @@ -380,6 +382,7 @@ protected: bool zoltanImbalanceTolSet_; double zoltanImbalanceTol_; + double zoltanPhgEdgeSizeThreshold_; std::string zoltanParams_; std::string metisParams_; diff --git a/opm/simulators/flow/GenericCpGridVanguard.cpp b/opm/simulators/flow/GenericCpGridVanguard.cpp index 233d7531f7..03bf389aeb 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.cpp +++ b/opm/simulators/flow/GenericCpGridVanguard.cpp @@ -158,11 +158,20 @@ doLoadBalance_(const Dune::EdgeWeightMethod edgeWeightsMethod, const int numJacobiBlocks, const bool enableEclOutput) { - if ((partitionMethod == Dune::PartitionMethod::zoltan - || partitionMethod == Dune::PartitionMethod::zoltanGoG) && !this->zoltanParams().empty()) - this->grid_->setPartitioningParams(setupZoltanParams(this->zoltanParams())); - if (partitionMethod == Dune::PartitionMethod::metis && !this->metisParams().empty()) - this->grid_->setPartitioningParams(setupMetisParams(this->metisParams())); + if (((partitionMethod == Dune::PartitionMethod::zoltan) || + (partitionMethod == Dune::PartitionMethod::zoltanGoG)) && + !this->zoltanParams().empty()) + { + this->grid_->setPartitioningParams + (setupZoltanParams(this->zoltanParams(), + this->zoltanPhgEdgeSizeThreshold())); + } + else if ((partitionMethod == Dune::PartitionMethod::metis) && + !this->metisParams().empty()) + { + this->grid_->setPartitioningParams + (setupMetisParams(this->metisParams())); + } const auto mpiSize = this->grid_->comm().size(); diff --git a/opm/simulators/flow/GenericCpGridVanguard.hpp b/opm/simulators/flow/GenericCpGridVanguard.hpp index 62c70f9f99..b9042bb9ab 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.hpp +++ b/opm/simulators/flow/GenericCpGridVanguard.hpp @@ -209,6 +209,7 @@ private: protected: virtual const std::string& zoltanParams() const = 0; + virtual double zoltanPhgEdgeSizeThreshold() const = 0; virtual const std::string& metisParams() const = 0; #endif // HAVE_MPI diff --git a/opm/simulators/utils/SetupPartitioningParams.cpp b/opm/simulators/utils/SetupPartitioningParams.cpp index febc176ae2..31c17c18c2 100644 --- a/opm/simulators/utils/SetupPartitioningParams.cpp +++ b/opm/simulators/utils/SetupPartitioningParams.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -99,6 +100,18 @@ namespace { namespace { + std::map + applyEdgeSizeThreshold(std::map&& params, + const std::optional& edgeSizeThreshold) + { + if (edgeSizeThreshold.has_value()) { + params.emplace("PHG_EDGE_SIZE_THRESHOLD", + fmt::format("{}", *edgeSizeThreshold)); + } + + return params; + } + std::map zoltanGraphParameters(std::string_view graphPackage) { @@ -108,9 +121,10 @@ namespace { }; } - auto zoltanGraphParameters() + auto zoltanGraphParameters(const std::optional& edgeSizeThreshold) { - return zoltanGraphParameters("PHG"); + return applyEdgeSizeThreshold(zoltanGraphParameters("PHG"), + edgeSizeThreshold); } auto zoltanScotchParameters() @@ -119,11 +133,12 @@ namespace { } std::map - zoltanHyperGraphParameters() + zoltanHyperGraphParameters(const std::optional& edgeSizeThreshold) { - return { + return applyEdgeSizeThreshold({ { "LB_METHOD", "HYPERGRAPH" }, - }; + { "HYPERGRAPH_PACKAGE", "PHG" }, + }, edgeSizeThreshold); } } // Anonymous namespace @@ -131,14 +146,15 @@ namespace { // =========================================================================== std::map -Opm::setupZoltanParams(const std::string& conf) +Opm::setupZoltanParams(const std::string& conf, + const std::optional& edgeSizeThreshold) { if (conf == "graph") { - return zoltanGraphParameters(); + return zoltanGraphParameters(edgeSizeThreshold); } else if (conf == "hypergraph") { - return zoltanHyperGraphParameters(); + return zoltanHyperGraphParameters(edgeSizeThreshold); } else if (conf == "scotch") { diff --git a/opm/simulators/utils/SetupPartitioningParams.hpp b/opm/simulators/utils/SetupPartitioningParams.hpp index b53be695f9..da6ff5f253 100644 --- a/opm/simulators/utils/SetupPartitioningParams.hpp +++ b/opm/simulators/utils/SetupPartitioningParams.hpp @@ -21,12 +21,47 @@ #define OPM_SETUP_PARTITIONING_PARAMS_HPP #include +#include #include namespace Opm { -std::map setupZoltanParams(const std::string& conf); -std::map setupMetisParams(const std::string& conf); +/// Form collection of Zoltan partitioning parameters from named configuration +/// +/// \param[in] conf Named Zoltan configuration. Must either be the name of a +/// JSON configuration file with the filename extension ".json", or one of +/// the known configuration names +/// +/// -* graph Generates configuration parameters for the "GRAPH" +/// load-balancing method, using the "PHG" graph package. +/// +/// -* hypergraph Generates configuration parameters for the "HYPERGRAPH" +/// load-balancing method. +/// +/// -* scotch Generates configuration parameters for the "GRAPH" +/// load-balancing method, using the "Scotch" graph package. +/// +/// \param[in] edgeSizeThreshold Low-level Zoltan partitioning control +/// parameter for when to omit a hyperedge in a hypergraph. Fraction in the +/// range [0,1] representing a threshold above which to omit discard +/// hyperedges. Used for conf="graph" and conf="hypergraph". Nullopt to +/// use the built-in default value. +/// +/// \return Collection of Zoltan partitioning parameters. +std::map +setupZoltanParams(const std::string& conf, + const std::optional& edgeSizeThreshold = {}); + +/// Form collection of METIS partitioning parameters from named configuration +/// +/// \param[in] conf Named METIS configuration. Must either be the name of a +/// JSON configuration file with the filename extension ".json", or the +/// known configuration name "default" which uses the built-in default +/// partitioning parameters. +/// +/// \return Collection of METIS partitioning parameters. +std::map +setupMetisParams(const std::string& conf); } // namespace Opm