Merge pull request #3244 from akva2/add_collective_comm

add collective communication to generic vanguard

ebos/eclgenericvanguard.cc

@@ -61,6 +61,7 @@ std::unique_ptr<EclipseState> EclGenericVanguard::externalEclState_;
 std::unique_ptr<Schedule> EclGenericVanguard::externalEclSchedule_;
 std::unique_ptr<SummaryConfig> EclGenericVanguard::externalEclSummaryConfig_;
 std::unique_ptr<UDQState> EclGenericVanguard::externalUDQState_;
+std::unique_ptr<EclGenericVanguard::CommunicationType> EclGenericVanguard::comm_;
 
 EclGenericVanguard::EclGenericVanguard()
     : python(std::make_shared<Python>())

ebos/eclgenericvanguard.hh

@@ -29,6 +29,10 @@
 
 #include <opm/grid/common/GridEnums.hpp>
 
+#include <dune/common/version.hh>
+#include <dune/common/parallel/collectivecommunication.hh>
+#include <dune/common/parallel/mpihelper.hh>
+
 #include <array>
 #include <memory>
 #include <optional>
@@ -57,6 +61,12 @@ class EclGenericVanguard {
 public:
     using ParallelWellStruct = std::vector<std::pair<std::string,bool>>;
 
+#if DUNE_VERSION_NEWER(DUNE_COMMON, 2, 7)
+    using CommunicationType = Dune::Communication<Dune::MPIHelper::MPICommunicator>;
+#else
+    using CommunicationType = Dune::CollectiveCommunication<Dune::MPIHelper::MPICommunicator>;
+#endif
+
     /*!
      * \brief Constructor.
      * \details Needs to be in compile unit.
@@ -260,6 +270,17 @@ public:
     const ParallelWellStruct& parallelWells() const
     { return parallelWells_; }
 
+    //! \brief Set global communication.
+    static void setCommunication(std::unique_ptr<CommunicationType> comm)
+    { comm_ = std::move(comm); }
+
+    //! \brief Obtain global communicator.
+    static CommunicationType& comm()
+    {
+        assert(comm_);
+        return *comm_;
+    }
+
 protected:
     void updateOutputDir_(std::string outputDir,
                           bool enableEclCompatFile);
@@ -279,7 +300,7 @@ protected:
     static std::unique_ptr<Schedule> externalEclSchedule_;
     static std::unique_ptr<SummaryConfig> externalEclSummaryConfig_;
     static std::unique_ptr<UDQState> externalUDQState_;
-
+    static std::unique_ptr<CommunicationType> comm_;
 
     std::string caseName_;
     std::string fileName_;

opm/simulators/flow/Main.hpp

@@ -98,11 +98,6 @@ namespace Opm {
     // with incorrect locale settings.
     resetLocale();
 
-# if HAVE_DUNE_FEM
-    Dune::Fem::MPIManager::initialize(argc, argv);
-# else
-    Dune::MPIHelper::instance(argc, argv);
-# endif
     FlowMainEbos<TypeTag> mainfunc(argc, argv, outputCout, outputFiles);
     return mainfunc.execute();
   }
@@ -125,7 +120,7 @@ namespace Opm
         using FlowMainEbosType = FlowMainEbos<Properties::TTag::EclFlowProblem>;
 
     public:
-        Main(int argc, char** argv) : argc_(argc), argv_(argv)  {  }
+        Main(int argc, char** argv) : argc_(argc), argv_(argv)  { initMPI();  }
 
         Main(const std::string &filename)
         {
@@ -135,6 +130,7 @@ namespace Opm
             saveArgs_[0] = const_cast<char *>(flowProgName_.c_str());
             saveArgs_[1] = const_cast<char *>(deckFilename_.c_str());
             argv_ = saveArgs_;
+            initMPI();
         }
 
         Main(int argc,
@@ -150,6 +146,26 @@ namespace Opm
             , schedule_(std::move(schedule))
             , summaryConfig_(std::move(summaryConfig))
         {
+          initMPI();
+        }
+
+        ~Main()
+        {
+            EclGenericVanguard::setCommunication(nullptr);
+
+#if HAVE_MPI && !HAVE_DUNE_FEM
+            MPI_Finalize();
+#endif
+        }
+
+        void initMPI()
+        {
+#if HAVE_DUNE_FEM
+            Dune::Fem::MPIManager::initialize(argc_, argv_);
+#elif HAVE_MPI
+            MPI_Init(&argc_, &argv_);
+#endif
+            EclGenericVanguard::setCommunication(std::make_unique<EclGenericVanguard::CommunicationType>());
         }
 
         int runDynamic()
@@ -356,17 +372,10 @@ namespace Opm
             externalSetupTimer.start();
 
             handleVersionCmdLine_(argc_, argv_);
-            // MPI setup.
 #if HAVE_DUNE_FEM
-            Dune::Fem::MPIManager::initialize(argc_, argv_);
             int mpiRank = Dune::Fem::MPIManager::rank();
 #else
-            // the design of the plain dune MPIHelper class is quite flawed: there is no way to
+            int mpiRank = EclGenericVanguard::comm().rank();
-            // get the instance without having the argc and argv parameters available and it is
-            // not possible to determine the MPI rank and size without an instance. (IOW: the
-            // rank() and size() methods are supposed to be static.)
-            const auto& mpiHelper = Dune::MPIHelper::instance(argc_, argv_);
-            int mpiRank = mpiHelper.rank();
 #endif
 
             // we always want to use the default locale, and thus spare us the trouble

opm/simulators/flow/python/simulators.hpp

@@ -50,8 +50,12 @@ private:
     bool hasRunInit_ = false;
     bool hasRunCleanup_ = false;
 
-    std::unique_ptr<Opm::FlowMainEbos<TypeTag>> mainEbos_;
+    // This *must* be declared before other pointers
+    // to simulator objects. This in order to deinitialize
+    // MPI at the correct time (ie after the other objects).
     std::unique_ptr<Opm::Main> main_;
+
+    std::unique_ptr<Opm::FlowMainEbos<TypeTag>> mainEbos_;
     Simulator *ebosSimulator_;
     std::unique_ptr<PyMaterialState<TypeTag>> materialState_;
 };