From 610c45aa77ede77144a9318a4a6aa02edd4a2b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A5rd=20Skaflestad?= Date: Mon, 18 Sep 2023 10:21:31 +0200 Subject: [PATCH] Add Support for Reading MPI Partitioning From File This commit introduces new, experimental support for loading a partitioning of the cells from a text file. The name of the file is passed into the simulator using the new, hidden, command line option --external-partition=filename and we perform some basic checking that the number of elements in the partition matches the number of cells in the CpGrid object. --- ebos/eclbasevanguard.hh | 19 +++++++ ebos/eclcpgridvanguard.hh | 53 +++++++++++++++++++ ebos/eclgenericcpgridvanguard.cc | 1 + ebos/eclgenericvanguard.hh | 6 +++ .../SimulatorFullyImplicitBlackoilEbos.hpp | 2 - 5 files changed, 79 insertions(+), 2 deletions(-) diff --git a/ebos/eclbasevanguard.hh b/ebos/eclbasevanguard.hh index 7d69a62c6..691b18383 100644 --- a/ebos/eclbasevanguard.hh +++ b/ebos/eclbasevanguard.hh @@ -119,6 +119,12 @@ struct ZoltanParams { using type = UndefinedProperty; }; +template +struct ExternalPartition +{ + using type = UndefinedProperty; +}; + template struct AllowDistributedWells { using type = UndefinedProperty; @@ -179,6 +185,12 @@ struct ZoltanParams { static constexpr auto value = "graph"; }; +template +struct ExternalPartition +{ + static constexpr auto* value = ""; +}; + template struct AllowDistributedWells { static constexpr bool value = false; @@ -268,6 +280,12 @@ public: "See https://sandialabs.github.io/Zoltan/ug_html/ug.html " "for available Zoltan options."); EWOMS_HIDE_PARAM(TypeTag, ZoltanParams); + EWOMS_REGISTER_PARAM(TypeTag, std::string, ExternalPartition, + "Name of file from which to load an externally generated " + "partitioning of the model's active cells for MPI " + "distribution purposes. If empty, the built-in partitioning " + "method will be employed."); + EWOMS_HIDE_PARAM(TypeTag, ExternalPartition); #endif EWOMS_REGISTER_PARAM(TypeTag, bool, AllowDistributedWells, "Allow the perforations of a well to be distributed to interior of multiple processes"); @@ -297,6 +315,7 @@ public: serialPartitioning_ = EWOMS_GET_PARAM(TypeTag, bool, SerialPartitioning); zoltanImbalanceTol_ = EWOMS_GET_PARAM(TypeTag, double, ZoltanImbalanceTol); zoltanParams_ = EWOMS_GET_PARAM(TypeTag, std::string, ZoltanParams); + externalPartitionFile_ = EWOMS_GET_PARAM(TypeTag, std::string, ExternalPartition); #endif enableDistributedWells_ = EWOMS_GET_PARAM(TypeTag, bool, AllowDistributedWells); ignoredKeywords_ = EWOMS_GET_PARAM(TypeTag, std::string, IgnoreKeywords); diff --git a/ebos/eclcpgridvanguard.hh b/ebos/eclcpgridvanguard.hh index 0b4b89fe4..08a314e96 100644 --- a/ebos/eclcpgridvanguard.hh +++ b/ebos/eclcpgridvanguard.hh @@ -38,11 +38,58 @@ #include #include +#include +#include #include +#include #include +#include #include #include +#include + +#if HAVE_MPI + +namespace Opm { namespace details { + class MPIPartitionFromFile + { + public: + explicit MPIPartitionFromFile(const std::filesystem::path& partitionFile) + : partitionFile_(partitionFile) + {} + + std::vector operator()(const Dune::CpGrid& grid) const; + + private: + std::filesystem::path partitionFile_{}; + }; + + inline std::vector + MPIPartitionFromFile::operator()(const Dune::CpGrid& grid) const + { + std::ifstream pfile { this->partitionFile_ }; + + auto partition = std::vector { + std::istream_iterator { pfile }, + std::istream_iterator {} + }; + + if (partition.size() != static_cast::size_type>(grid.size(0))) { + throw std::invalid_argument { + fmt::format("Partition file '{}' with {} values does " + "not match CpGrid instance with {} cells", + this->partitionFile_.generic_string(), + partition.size(), grid.size(0)) + }; + } + + return partition; + } +}} // namespace Opm::details + +#endif // HAVE_MPI + namespace Opm { template class EclCpGridVanguard; @@ -224,6 +271,12 @@ public: void loadBalance() { #if HAVE_MPI + if (const auto& extPFile = this->externalPartitionFile(); + !extPFile.empty() && (extPFile != "none")) + { + this->setExternalLoadBalancer(details::MPIPartitionFromFile { extPFile }); + } + this->doLoadBalance_(this->edgeWeightsMethod(), this->ownersFirst(), this->serialPartitioning(), this->enableDistributedWells(), this->zoltanImbalanceTol(), this->gridView(), diff --git a/ebos/eclgenericcpgridvanguard.cc b/ebos/eclgenericcpgridvanguard.cc index 35eaa4037..76161703b 100644 --- a/ebos/eclgenericcpgridvanguard.cc +++ b/ebos/eclgenericcpgridvanguard.cc @@ -52,6 +52,7 @@ #include #endif //HAVE_DUNE_FEM +#include #include #include #include diff --git a/ebos/eclgenericvanguard.hh b/ebos/eclgenericvanguard.hh index 058e59fb8..8728a85aa 100644 --- a/ebos/eclgenericvanguard.hh +++ b/ebos/eclgenericvanguard.hh @@ -228,6 +228,11 @@ public: */ double zoltanImbalanceTol() const { return zoltanImbalanceTol_; } + + const std::string& externalPartitionFile() const + { + return this->externalPartitionFile_; + } #endif /*! @@ -292,6 +297,7 @@ protected: bool serialPartitioning_; double zoltanImbalanceTol_; std::string zoltanParams_; + std::string externalPartitionFile_{}; #endif bool enableDistributedWells_; std::string ignoredKeywords_; diff --git a/opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp b/opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp index 01c2b1623..43bc1fcdc 100644 --- a/opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp +++ b/opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp @@ -97,7 +97,6 @@ struct LoadFile using type = UndefinedProperty; }; - template struct EnableTerminalOutput { static constexpr bool value = true; @@ -135,7 +134,6 @@ struct LoadFile static constexpr auto* value = ""; }; - template struct LoadStep {