Merge pull request #4117 from akva2/zoltan_params

added: command line option for zoltan customization

CMakeLists_files.cmake

@@ -144,9 +144,10 @@ if(HAVE_AMGCL)
   endif()
 endif()
 if(MPI_FOUND)
-  list(APPEND MAIN_SOURCE_FILES opm/simulators/utils/ParallelEclipseState.cpp
+  list(APPEND MAIN_SOURCE_FILES opm/simulators/utils/MPIPacker.cpp
+                                opm/simulators/utils/ParallelEclipseState.cpp
                                 opm/simulators/utils/ParallelSerialization.cpp
-                                opm/simulators/utils/MPIPacker.cpp)
+                                opm/simulators/utils/SetupZoltanParams.cpp)
 endif()
 
 # originally generated with the command:

ebos/eclbasevanguard.hh

@@ -107,6 +107,11 @@ struct ZoltanImbalanceTol {
     using type = UndefinedProperty;
 };
 
+template<class TypeTag, class MyTypeTag>
+struct ZoltanParams {
+    using type = UndefinedProperty;
+};
+
 template<class TypeTag, class MyTypeTag>
 struct AllowDistributedWells {
     using type = UndefinedProperty;
@@ -162,6 +167,11 @@ struct ZoltanImbalanceTol<TypeTag, TTag::EclBaseVanguard> {
     static constexpr double value = 1.1;
 };
 
+template<class TypeTag>
+struct ZoltanParams<TypeTag,TTag::EclBaseVanguard> {
+    static constexpr auto value = "graph";
+};
+
 template<class TypeTag>
 struct AllowDistributedWells<TypeTag, TTag::EclBaseVanguard> {
     static constexpr bool value = false;
@@ -235,10 +245,14 @@ public:
 
         EWOMS_REGISTER_PARAM(TypeTag, bool, OwnerCellsFirst,
                              "Order cells owned by rank before ghost/overlap cells.");
+#if HAVE_MPI
         EWOMS_REGISTER_PARAM(TypeTag, bool, SerialPartitioning,
                              "Perform partitioning for parallel runs on a single process.");
         EWOMS_REGISTER_PARAM(TypeTag, double, ZoltanImbalanceTol,
                              "Tolerable imbalance of the loadbalancing provided by Zoltan (default: 1.1).");
+        EWOMS_REGISTER_PARAM(TypeTag, std::string, ZoltanParams,
+                             "Configuration of Zoltan partitioner. Valid options are: graph (default)), hypergrah or scotch. Alternatively, you can request a configuration to be read from a JSON file by giving the filename here, ending with '.json.'");
+#endif
         EWOMS_REGISTER_PARAM(TypeTag, bool, AllowDistributedWells,
                              "Allow the perforations of a well to be distributed to interior of multiple processes");
         // register here for the use in the tests without BlackoildModelParametersEbos
@@ -263,8 +277,11 @@ public:
 #endif
 
         ownersFirst_ = EWOMS_GET_PARAM(TypeTag, bool, OwnerCellsFirst);
+#if HAVE_MPI
         serialPartitioning_ = EWOMS_GET_PARAM(TypeTag, bool, SerialPartitioning);
         zoltanImbalanceTol_ = EWOMS_GET_PARAM(TypeTag, double, ZoltanImbalanceTol);
+        zoltanParams_ = EWOMS_GET_PARAM(TypeTag, std::string, ZoltanParams);
+#endif
         enableDistributedWells_ = EWOMS_GET_PARAM(TypeTag, bool, AllowDistributedWells);
         ignoredKeywords_ = EWOMS_GET_PARAM(TypeTag, std::string, IgnoreKeywords);
         eclStrictParsing_ = EWOMS_GET_PARAM(TypeTag, bool, EclStrictParsing);

ebos/eclcpgridvanguard.hh

@@ -197,6 +197,13 @@ protected:
        return globalTrans_->transmissibility(I,J);
     }
 
+#if HAVE_MPI
+    const std::string& zoltanParams() const override
+    {
+        return this->zoltanParams_;
+    }
+#endif
+
     // removing some connection located in inactive grid cells
     void filterConnections_()
     {

ebos/eclgenericcpgridvanguard.cc

@@ -32,6 +32,7 @@
 #include <opm/simulators/utils/ParallelEclipseState.hpp>
 #include <opm/simulators/utils/ParallelSerialization.hpp>
 #include <opm/simulators/utils/PropsCentroidsDataHandle.hpp>
+#include <opm/simulators/utils/SetupZoltanParams.hpp>
 
 #include <opm/grid/cpgrid/GridHelpers.hpp>
 
@@ -97,6 +98,9 @@ doLoadBalance_(const Dune::EdgeWeightMethod            edgeWeightsMethod,
                EclGenericVanguard::ParallelWellStruct& parallelWells,
                const int                               numJacobiBlocks)
 {
+    if (!this->zoltanParams().empty())
+        this->grid_->setZoltanParams(setupZoltanParams(this->zoltanParams()));
+
     const auto mpiSize = this->grid_->comm().size();
 
     const auto partitionJacobiBlocks =

ebos/eclgenericcpgridvanguard.hh

@@ -164,6 +164,8 @@ private:
                         EclGenericVanguard::ParallelWellStruct& parallelWells);
 
 protected:
+    virtual const std::string& zoltanParams() const = 0;
+
 #endif  // HAVE_MPI
 
     void allocCartMapper();

ebos/eclgenericvanguard.hh

@@ -218,6 +218,7 @@ public:
     bool ownersFirst() const
     { return ownersFirst_; }
 
+#if HAVE_MPI
     /*!
      * \brief Parameter that decides if partitioning for parallel runs
      *        should be performed on a single process only.
@@ -230,6 +231,7 @@ public:
      */
     double zoltanImbalanceTol() const
     { return zoltanImbalanceTol_; }
+#endif
 
     /*!
      * \brief Whether perforations of a well might be distributed.
@@ -281,8 +283,11 @@ protected:
 #endif  // HAVE_OPENCL
 
     bool ownersFirst_;
+#if HAVE_MPI
     bool serialPartitioning_;
     double zoltanImbalanceTol_;
+    std::string zoltanParams_;
+#endif
     bool enableDistributedWells_;
     std::string ignoredKeywords_;
     bool eclStrictParsing_;

opm/simulators/utils/SetupZoltanParams.cpp

@@ -0,0 +1,60 @@
+/*
+  Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+#include <opm/simulators/utils/SetupZoltanParams.hpp>
+
+#include <opm/common/OpmLog/OpmLog.hpp>
+
+#include <boost/property_tree/json_parser.hpp>
+
+namespace Opm
+{
+
+
+std::map<std::string,std::string> setupZoltanParams(const std::string& conf)
+{
+    std::map<std::string,std::string> result;
+
+    if (conf == "scotch") {
+        result.emplace("LB_METHOD", "GRAPH");
+        result.emplace("GRAPH_PACKAGE", "Scotch");
+    } else if (conf == "hypergraph") {
+        result.emplace("LB_METHOD", "HYPERGRAPH");
+    } else if (conf == "graph") {
+        result.emplace("LB_METHOD", "GRAPH");
+        result.emplace("GRAPH_PACKAGE", "PHG");
+    } else { // json file
+        boost::property_tree::ptree tree;
+        try {
+            boost::property_tree::read_json(conf, tree);
+        } catch (boost::property_tree::json_parser::json_parser_error& err) {
+            OpmLog::error(err.what());
+        }
+        for (const auto& node : tree) {
+            auto value = node.second.get_value_optional<std::string>();
+            if (value)
+              result.insert_or_assign(node.first, *value);
+        }
+    }
+
+    return result;
+}
+
+} // namespace Opm

opm/simulators/utils/SetupZoltanParams.hpp

@@ -0,0 +1,32 @@
+/*
+  Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef OPM_SETUP_ZOLTAN_PARAMS_HPP
+#define OPM_SETUP_ZOLTAN_PARAMS_HPP
+
+#include <map>
+#include <string>
+
+namespace Opm {
+
+std::map<std::string,std::string> setupZoltanParams(const std::string& conf);
+
+} // namespace Opm
+
+#endif // OPM_SETUP_ZOLTAN_PARAMS_HPP