diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index 16b247c92..347a1e9a9 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -144,9 +144,10 @@ if(HAVE_AMGCL) endif() endif() if(MPI_FOUND) - list(APPEND MAIN_SOURCE_FILES opm/simulators/utils/ParallelEclipseState.cpp + list(APPEND MAIN_SOURCE_FILES opm/simulators/utils/MPIPacker.cpp + opm/simulators/utils/ParallelEclipseState.cpp opm/simulators/utils/ParallelSerialization.cpp - opm/simulators/utils/MPIPacker.cpp) + opm/simulators/utils/SetupZoltanParams.cpp) endif() # originally generated with the command: diff --git a/ebos/eclbasevanguard.hh b/ebos/eclbasevanguard.hh index 8298d7a2c..64f81c30f 100644 --- a/ebos/eclbasevanguard.hh +++ b/ebos/eclbasevanguard.hh @@ -107,6 +107,11 @@ struct ZoltanImbalanceTol { using type = UndefinedProperty; }; +template +struct ZoltanParams { + using type = UndefinedProperty; +}; + template struct AllowDistributedWells { using type = UndefinedProperty; @@ -162,6 +167,11 @@ struct ZoltanImbalanceTol { static constexpr double value = 1.1; }; +template +struct ZoltanParams { + static constexpr auto value = "graph"; +}; + template struct AllowDistributedWells { static constexpr bool value = false; @@ -235,10 +245,14 @@ public: EWOMS_REGISTER_PARAM(TypeTag, bool, OwnerCellsFirst, "Order cells owned by rank before ghost/overlap cells."); +#if HAVE_MPI EWOMS_REGISTER_PARAM(TypeTag, bool, SerialPartitioning, "Perform partitioning for parallel runs on a single process."); EWOMS_REGISTER_PARAM(TypeTag, double, ZoltanImbalanceTol, "Tolerable imbalance of the loadbalancing provided by Zoltan (default: 1.1)."); + EWOMS_REGISTER_PARAM(TypeTag, std::string, ZoltanParams, + "Configuration of Zoltan partitioner. Valid options are: graph (default)), hypergrah or scotch. Alternatively, you can request a configuration to be read from a JSON file by giving the filename here, ending with '.json.'"); +#endif EWOMS_REGISTER_PARAM(TypeTag, bool, AllowDistributedWells, "Allow the perforations of a well to be distributed to interior of multiple processes"); // register here for the use in the tests without BlackoildModelParametersEbos @@ -263,8 +277,11 @@ public: #endif ownersFirst_ = EWOMS_GET_PARAM(TypeTag, bool, OwnerCellsFirst); +#if HAVE_MPI serialPartitioning_ = EWOMS_GET_PARAM(TypeTag, bool, SerialPartitioning); zoltanImbalanceTol_ = EWOMS_GET_PARAM(TypeTag, double, ZoltanImbalanceTol); + zoltanParams_ = EWOMS_GET_PARAM(TypeTag, std::string, ZoltanParams); +#endif enableDistributedWells_ = EWOMS_GET_PARAM(TypeTag, bool, AllowDistributedWells); ignoredKeywords_ = EWOMS_GET_PARAM(TypeTag, std::string, IgnoreKeywords); eclStrictParsing_ = EWOMS_GET_PARAM(TypeTag, bool, EclStrictParsing); diff --git a/ebos/eclcpgridvanguard.hh b/ebos/eclcpgridvanguard.hh index 5d9920697..ce6c55650 100644 --- a/ebos/eclcpgridvanguard.hh +++ b/ebos/eclcpgridvanguard.hh @@ -197,6 +197,13 @@ protected: return globalTrans_->transmissibility(I,J); } +#if HAVE_MPI + const std::string& zoltanParams() const override + { + return this->zoltanParams_; + } +#endif + // removing some connection located in inactive grid cells void filterConnections_() { diff --git a/ebos/eclgenericcpgridvanguard.cc b/ebos/eclgenericcpgridvanguard.cc index 404c02f71..2fc206bf5 100644 --- a/ebos/eclgenericcpgridvanguard.cc +++ b/ebos/eclgenericcpgridvanguard.cc @@ -32,6 +32,7 @@ #include #include #include +#include #include @@ -97,6 +98,9 @@ doLoadBalance_(const Dune::EdgeWeightMethod edgeWeightsMethod, EclGenericVanguard::ParallelWellStruct& parallelWells, const int numJacobiBlocks) { + if (!this->zoltanParams().empty()) + this->grid_->setZoltanParams(setupZoltanParams(this->zoltanParams())); + const auto mpiSize = this->grid_->comm().size(); const auto partitionJacobiBlocks = diff --git a/ebos/eclgenericcpgridvanguard.hh b/ebos/eclgenericcpgridvanguard.hh index de0e4f1af..c873b410b 100644 --- a/ebos/eclgenericcpgridvanguard.hh +++ b/ebos/eclgenericcpgridvanguard.hh @@ -164,6 +164,8 @@ private: EclGenericVanguard::ParallelWellStruct& parallelWells); protected: + virtual const std::string& zoltanParams() const = 0; + #endif // HAVE_MPI void allocCartMapper(); diff --git a/ebos/eclgenericvanguard.hh b/ebos/eclgenericvanguard.hh index 2500e0854..31231e93f 100644 --- a/ebos/eclgenericvanguard.hh +++ b/ebos/eclgenericvanguard.hh @@ -218,6 +218,7 @@ public: bool ownersFirst() const { return ownersFirst_; } +#if HAVE_MPI /*! * \brief Parameter that decides if partitioning for parallel runs * should be performed on a single process only. @@ -230,6 +231,7 @@ public: */ double zoltanImbalanceTol() const { return zoltanImbalanceTol_; } +#endif /*! * \brief Whether perforations of a well might be distributed. @@ -281,8 +283,11 @@ protected: #endif // HAVE_OPENCL bool ownersFirst_; +#if HAVE_MPI bool serialPartitioning_; double zoltanImbalanceTol_; + std::string zoltanParams_; +#endif bool enableDistributedWells_; std::string ignoredKeywords_; bool eclStrictParsing_; diff --git a/opm/simulators/utils/SetupZoltanParams.cpp b/opm/simulators/utils/SetupZoltanParams.cpp new file mode 100644 index 000000000..054d39ef4 --- /dev/null +++ b/opm/simulators/utils/SetupZoltanParams.cpp @@ -0,0 +1,60 @@ +/* + Copyright 2022 SINTEF Digital, Mathematics and Cybernetics. + + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ + +#include +#include + +#include + +#include + +namespace Opm +{ + + +std::map setupZoltanParams(const std::string& conf) +{ + std::map result; + + if (conf == "scotch") { + result.emplace("LB_METHOD", "GRAPH"); + result.emplace("GRAPH_PACKAGE", "Scotch"); + } else if (conf == "hypergraph") { + result.emplace("LB_METHOD", "HYPERGRAPH"); + } else if (conf == "graph") { + result.emplace("LB_METHOD", "GRAPH"); + result.emplace("GRAPH_PACKAGE", "PHG"); + } else { // json file + boost::property_tree::ptree tree; + try { + boost::property_tree::read_json(conf, tree); + } catch (boost::property_tree::json_parser::json_parser_error& err) { + OpmLog::error(err.what()); + } + for (const auto& node : tree) { + auto value = node.second.get_value_optional(); + if (value) + result.insert_or_assign(node.first, *value); + } + } + + return result; +} + +} // namespace Opm diff --git a/opm/simulators/utils/SetupZoltanParams.hpp b/opm/simulators/utils/SetupZoltanParams.hpp new file mode 100644 index 000000000..1e77c3a78 --- /dev/null +++ b/opm/simulators/utils/SetupZoltanParams.hpp @@ -0,0 +1,32 @@ +/* + Copyright 2022 SINTEF Digital, Mathematics and Cybernetics. + + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ + +#ifndef OPM_SETUP_ZOLTAN_PARAMS_HPP +#define OPM_SETUP_ZOLTAN_PARAMS_HPP + +#include +#include + +namespace Opm { + +std::map setupZoltanParams(const std::string& conf); + +} // namespace Opm + +#endif // OPM_SETUP_ZOLTAN_PARAMS_HPP