flow_ebos*: make it build and (the sequential part) work if dune-fem is available

in particular, this implied some changes to the MPI initialization
code. since dune-fem's GridPart class currently has issues with
CpGrid's implementation of loadBalance(), parallel computations still
do not work if dune-fem is around, but at least sequential ones now
do even if MPI is enabled.

examples/flow.cpp

@@ -27,6 +27,8 @@
 
 #include <opm/core/utility/parameters/ParameterGroup.hpp>
 
+#include <opm/common/ResetLocale.hpp>
+
 #include <opm/parser/eclipse/Deck/Deck.hpp>
 #include <opm/parser/eclipse/Parser/Parser.hpp>
 #include <opm/parser/eclipse/Parser/ParseContext.hpp>
@@ -38,7 +40,6 @@
 // Define making clear that the simulator supports AMG
 #define FLOW_SUPPORT_AMG 1
 
-#include <opm/material/densead/Evaluation.hpp>
 #include <ewoms/models/blackoil/blackoiltwophaseindices.hh>
 
 #include <opm/autodiff/DuneMatrix.hpp>
@@ -116,17 +117,30 @@ namespace detail
 int main(int argc, char** argv)
 {
     // MPI setup.
-    // Must ensure an instance of the helper is created to initialise MPI.
+#if HAVE_DUNE_FEM
-    // For a build without MPI the Dune::FakeMPIHelper is used, so rank will
+    Dune::Fem::MPIManager::initialize(argc, argv);
-    // be 0 and size 1.
+    int mpiRank = Dune::Fem::MPIManager::rank();
-    const Dune::MPIHelper& mpi_helper = Dune::MPIHelper::instance(argc, argv);
+#else
-    const bool outputCout = mpi_helper.rank() == 0;
+    // the design of the plain dune MPIHelper class is quite flawed: there is no way to
+    // get the instance without having the argc and argv parameters available and it is
+    // not possible to determine the MPI rank and size without an instance. (IOW: the
+    // rank() and size() methods are supposed to be static.)
+    const auto& mpiHelper = Dune::MPIHelper::instance(argc, argv);
+    int mpiRank = mpiHelper.rank();
+#endif
+
+    const bool outputCout = (mpiRank == 0);
+
+    // we always want to use the default locale, and thus spare us the trouble
+    // with incorrect locale settings.
+    Opm::resetLocale();
 
     Opm::ParameterGroup param(argc, argv, false, outputCout);
 
     // See if a deck was specified on the command line.
     if (!param.unhandledArguments().empty()) {
         if (param.unhandledArguments().size() != 1) {
+            if (outputCout)
                 std::cerr << "You can only specify a single input deck on the command line.\n";
             return EXIT_FAILURE;
         } else {
@@ -137,6 +151,7 @@ int main(int argc, char** argv)
 
     // We must have an input deck. Grid and props will be read from that.
     if (!param.has("deck_filename")) {
+        if (outputCout)
             std::cerr << "This program must be run with an input deck.\n"
                 "Specify the deck filename either\n"
                 "    a) as a command line argument by itself\n"
@@ -160,8 +175,8 @@ int main(int argc, char** argv)
         Opm::ParseContext parseContext(tmp);
 
         std::shared_ptr<Opm::Deck> deck = std::make_shared< Opm::Deck >( parser.parseFile(deckFilename , parseContext) );
-        Opm::checkDeck(*deck, parser);
         if ( outputCout ) {
+            Opm::checkDeck(*deck, parser);
             Opm::MissingFeatures::checkKeywords(*deck);
         }
 
@@ -171,7 +186,7 @@ int main(int argc, char** argv)
         Opm::Runspec runspec( *deck );
         const auto& phases = runspec.phases();
 
-        // Twophase case
+        // Twophase cases
         if( phases.size() == 2 ) {
             // oil-gas
             if (phases.active( Opm::Phase::GAS ))
@@ -186,6 +201,7 @@ int main(int argc, char** argv)
                 return mainfunc.execute(argc, argv, deck, eclipseState );
             }
             else {
+                if (outputCout)
                     std::cerr << "No suitable configuration found, valid are Twophase (oilwater and oilgas), polymer, solvent, or blackoil" << std::endl;
                 return EXIT_FAILURE;
             }
@@ -209,14 +225,17 @@ int main(int argc, char** argv)
         }
         else
         {
+            if (outputCout)
                 std::cerr << "No suitable configuration found, valid are Twophase, polymer, solvent, or blackoil" << std::endl;
             return EXIT_FAILURE;
         }
     }
     catch (const std::invalid_argument& e)
     {
+        if (outputCout) {
             std::cerr << "Failed to create valid EclipseState object." << std::endl;
             std::cerr << "Exception caught: " << e.what() << std::endl;
+        }
         throw;
     }
 

examples/flow_ebos.cpp

@@ -27,16 +27,30 @@
 // Define making clear that the simulator supports AMG
 #define FLOW_SUPPORT_AMG 1
 
-#include <opm/material/densead/Evaluation.hpp>
+#include <opm/common/ResetLocale.hpp>
-#include <opm/autodiff/DuneMatrix.hpp>
 #include <dune/grid/CpGrid.hpp>
 #include <opm/autodiff/SimulatorFullyImplicitBlackoilEbos.hpp>
 #include <opm/autodiff/FlowMainEbos.hpp>
 
+#if HAVE_DUNE_FEM
+#include <dune/fem/misc/mpimanager.hh>
+#else
+#include <dune/common/parallel/mpihelper.hh>
+#endif
 
 // ----------------- Main program -----------------
 int main(int argc, char** argv)
 {
+    // we always want to use the default locale, and thus spare us the trouble
+    // with incorrect locale settings.
+    Opm::resetLocale();
+
+#if HAVE_DUNE_FEM
+    Dune::Fem::MPIManager::initialize(argc, argv);
+#else
+    Dune::MPIHelper::instance(argc, argv);
+#endif
+
     Opm::FlowMainEbos<TTAG(EclFlowProblem)> mainfunc;
     return mainfunc.execute(argc, argv);
 }

examples/flow_ebos_2p.cpp

@@ -25,14 +25,19 @@
 // Define making clear that the simulator supports AMG
 #define FLOW_SUPPORT_AMG 1
 
-#include <opm/material/densead/Evaluation.hpp>
+#include <opm/common/ResetLocale.hpp>
 #include <ewoms/models/blackoil/blackoiltwophaseindices.hh>
 
-#include <opm/autodiff/DuneMatrix.hpp>
 #include <dune/grid/CpGrid.hpp>
 #include <opm/autodiff/SimulatorFullyImplicitBlackoilEbos.hpp>
 #include <opm/autodiff/FlowMainEbos.hpp>
 
+#if HAVE_DUNE_FEM
+#include <dune/fem/misc/mpimanager.hh>
+#else
+#include <dune/common/parallel/mpihelper.hh>
+#endif
+
 namespace Ewoms {
 namespace Properties {
 NEW_TYPE_TAG(EclFlowTwoPhaseProblem, INHERITS_FROM(EclFlowProblem));
@@ -44,6 +49,16 @@ SET_TYPE_PROP(EclFlowTwoPhaseProblem, Indices,
 // ----------------- Main program -----------------
 int main(int argc, char** argv)
 {
+    // we always want to use the default locale, and thus spare us the trouble
+    // with incorrect locale settings.
+    Opm::resetLocale();
+
+#if HAVE_DUNE_FEM
+    Dune::Fem::MPIManager::initialize(argc, argv);
+#else
+    Dune::MPIHelper::instance(argc, argv);
+#endif
+
     Opm::FlowMainEbos<TTAG(EclFlowTwoPhaseProblem)> mainfunc;
     return mainfunc.execute(argc, argv);
 }

examples/flow_ebos_polymer.cpp

@@ -22,12 +22,17 @@
 #include "config.h"
 #endif // HAVE_CONFIG_H
 
-#include <opm/material/densead/Evaluation.hpp>
+#include <opm/common/ResetLocale.hpp>
-#include <opm/autodiff/DuneMatrix.hpp>
 #include <dune/grid/CpGrid.hpp>
 #include <opm/autodiff/SimulatorFullyImplicitBlackoilEbos.hpp>
 #include <opm/autodiff/FlowMainEbos.hpp>
 
+#if HAVE_DUNE_FEM
+#include <dune/fem/misc/mpimanager.hh>
+#else
+#include <dune/common/parallel/mpihelper.hh>
+#endif
+
 namespace Ewoms {
 namespace Properties {
 NEW_TYPE_TAG(EclFlowPolymerProblem, INHERITS_FROM(EclFlowProblem));
@@ -37,6 +42,18 @@ SET_BOOL_PROP(EclFlowPolymerProblem, EnablePolymer, true);
 // ----------------- Main program -----------------
 int main(int argc, char** argv)
 {
+    // we always want to use the default locale, and thus spare us the trouble
+    // with incorrect locale settings.
+    Opm::resetLocale();
+
+    // initialize MPI, finalize is done automatically on exit
+#if HAVE_DUNE_FEM
+    Dune::Fem::MPIManager::initialize(argc, argv);
+    const int myRank = Dune::Fem::MPIManager::rank();
+#else
+    const int myRank = Dune::MPIHelper::instance(argc, argv).rank();
+#endif
+
     Opm::FlowMainEbos<TTAG(EclFlowPolymerProblem)> mainfunc;
     return mainfunc.execute(argc, argv);
 }

examples/flow_ebos_solvent.cpp

@@ -22,12 +22,17 @@
 #include "config.h"
 #endif // HAVE_CONFIG_H
 
-#include <opm/material/densead/Evaluation.hpp>
+#include <opm/common/ResetLocale.hpp>
-#include <opm/autodiff/DuneMatrix.hpp>
 #include <dune/grid/CpGrid.hpp>
 #include <opm/autodiff/SimulatorFullyImplicitBlackoilEbos.hpp>
 #include <opm/autodiff/FlowMainEbos.hpp>
 
+#if HAVE_DUNE_FEM
+#include <dune/fem/misc/mpimanager.hh>
+#else
+#include <dune/common/parallel/mpihelper.hh>
+#endif
+
 namespace Ewoms {
 namespace Properties {
 NEW_TYPE_TAG(EclFlowSolventProblem, INHERITS_FROM(EclFlowProblem));
@@ -37,6 +42,18 @@ SET_BOOL_PROP(EclFlowSolventProblem, EnableSolvent, true);
 // ----------------- Main program -----------------
 int main(int argc, char** argv)
 {
+    // we always want to use the default locale, and thus spare us the trouble
+    // with incorrect locale settings.
+    Opm::resetLocale();
+
+    // initialize MPI, finalize is done automatically on exit
+#if HAVE_DUNE_FEM
+    Dune::Fem::MPIManager::initialize(argc, argv);
+    const int myRank = Dune::Fem::MPIManager::rank();
+#else
+    const int myRank = Dune::MPIHelper::instance(argc, argv).rank();
+#endif
+
     Opm::FlowMainEbos<TTAG(EclFlowSolventProblem)> mainfunc;
     return mainfunc.execute(argc, argv);
 }

opm/autodiff/FlowMainEbos.hpp

@@ -45,7 +45,6 @@
 #include <opm/common/OpmLog/OpmLog.hpp>
 #include <opm/common/OpmLog/EclipsePRTLog.hpp>
 #include <opm/common/OpmLog/LogUtil.hpp>
-#include <opm/common/ResetLocale.hpp>
 
 #include <opm/parser/eclipse/Deck/Deck.hpp>
 #include <opm/parser/eclipse/Parser/Parser.hpp>
@@ -55,7 +54,11 @@
 #include <opm/parser/eclipse/EclipseState/InitConfig/InitConfig.hpp>
 #include <opm/parser/eclipse/EclipseState/checkDeck.hpp>
 
-#include <ewoms/version.hh>
+#if HAVE_DUNE_FEM
+#include <dune/fem/misc/mpimanager.hh>
+#else
+#include <dune/common/parallel/mpihelper.hh>
+#endif
 
 namespace Opm
 {
@@ -76,8 +79,8 @@ namespace Opm
         typedef typename GET_PROP(TypeTag, MaterialLaw)::EclMaterialLawManager MaterialLawManager;
         typedef typename GET_PROP_TYPE(TypeTag, Simulator) EbosSimulator;
         typedef typename GET_PROP_TYPE(TypeTag, SimulatorParameter) EbosSimulatorParameter;
-        typedef typename GET_PROP_TYPE(TypeTag, ElementMapper) ElementMapper;
         typedef typename GET_PROP_TYPE(TypeTag, Grid) Grid;
+        typedef typename GET_PROP_TYPE(TypeTag, GridView) GridView;
         typedef typename GET_PROP_TYPE(TypeTag, Problem) Problem;
         typedef typename GET_PROP_TYPE(TypeTag, Scalar) Scalar;
         typedef typename GET_PROP_TYPE(TypeTag, FluidSystem) FluidSystem;
@@ -96,10 +99,6 @@ namespace Opm
                     std::shared_ptr<Opm::EclipseState> eclipseState = std::shared_ptr<Opm::EclipseState>() )
         {
             try {
-                // we always want to use the default locale, and thus spare us the trouble
-                // with incorrect locale settings.
-                resetLocale();
-
                 setupParallelism(argc, argv);
                 printStartupMessage();
                 const bool ok = setupParameters(argc, argv);
@@ -149,13 +148,16 @@ namespace Opm
     protected:
         void setupParallelism(int argc, char** argv)
         {
-            // MPI setup.
+            // determine the rank of the current process and the number of processes
-            // Must ensure an instance of the helper is created to initialise MPI.
+            // involved in the simulation. MPI must have already been initialized here.
-            // For a build without MPI the Dune::FakeMPIHelper is used, so rank will
+#if HAVE_MPI
-            // be 0 and size 1.
+            MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank_);
-            const Dune::MPIHelper& mpi_helper = Dune::MPIHelper::instance(argc, argv);
+            int mpi_size;
-            mpi_rank_ = mpi_helper.rank();
+            MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
-            const int mpi_size = mpi_helper.size();
+#else
+            mpi_rank_ = 0;
+            const int mpi_size = 1;
+#endif
             output_cout_ = ( mpi_rank_ == 0 );
             must_distribute_ = ( mpi_size > 1 );
 
@@ -808,6 +810,8 @@ namespace Opm
 
             const Grid& globalGrid = this->globalGrid();
             const auto& globalGridView = globalGrid.leafGridView();
+            typedef typename Grid::LeafGridView GridView;
+            typedef Dune::MultipleCodimMultipleGeomTypeMapper<GridView, Dune::MCMGElementLayout> ElementMapper;
             ElementMapper globalElemMapper(globalGridView);
             const auto& cartesianCellIdx = globalGrid.globalCell();
 
@@ -881,6 +885,8 @@ namespace Opm
 
             const Grid& globalGrid = this->globalGrid();
             const auto& globalGridView = globalGrid.leafGridView();
+            typedef typename Grid::LeafGridView GridView;
+            typedef Dune::MultipleCodimMultipleGeomTypeMapper<GridView, Dune::MCMGElementLayout> ElementMapper;
             ElementMapper globalElemMapper(globalGridView);
 
             const auto* globalTrans = &(ebosSimulator_->gridManager().globalTransmissibility());