From 05c0c4b6e8c407f0d9036de7236ddfdff2a245fe Mon Sep 17 00:00:00 2001
From: Arne Morten Kvarving <arne.morten.kvarving@sintef.no>
Date: Fri, 24 Jan 2020 12:30:22 +0100
Subject: [PATCH] Use the ParallelEclipseState and ParallelFieldPropsManager

with these in place, we now only parse the deck on the root process.
---
 ebos/eclcpgridvanguard.hh            | 13 +++++++++++-
 flow/flow.cpp                        | 30 ++++++++++++++++++++--------
 opm/simulators/flow/FlowMainEbos.hpp |  8 ++++++--
 3 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/ebos/eclcpgridvanguard.hh b/ebos/eclcpgridvanguard.hh
index 5ac5decdc..67a60ead1 100644
--- a/ebos/eclcpgridvanguard.hh
+++ b/ebos/eclcpgridvanguard.hh
@@ -33,6 +33,7 @@
 
 #include <opm/grid/CpGrid.hpp>
 #include <opm/grid/cpgrid/GridHelpers.hpp>
+#include <opm/simulators/utils/ParallelEclipseState.hpp>
 
 #include <dune/grid/common/mcmgmapper.hh>
 
@@ -206,8 +207,18 @@ public:
 #endif
 
         cartesianIndexMapper_.reset(new CartesianIndexMapper(*grid_));
-
         this->updateGridView_();
+        if (mpiSize > 1) {
+            std::vector<int> cartIndices;
+            cartIndices.reserve(grid_->numCells());
+            auto locElemIt = this->gridView().template begin</*codim=*/0>();
+            const auto& locElemEndIt = this->gridView().template end</*codim=*/0>();
+            for (; locElemIt != locElemEndIt; ++locElemIt) {
+                cartIndices.push_back(cartesianIndexMapper_->cartesianIndex(locElemIt->index()));
+            }
+            static_cast<ParallelEclipseState&>(this->eclState()).setupLocalProps(cartIndices);
+            static_cast<ParallelEclipseState&>(this->eclState()).switchToDistributedProps();
+        }
     }
 
     /*!
diff --git a/flow/flow.cpp b/flow/flow.cpp
index 7f6d07605..3ebd2ca8b 100644
--- a/flow/flow.cpp
+++ b/flow/flow.cpp
@@ -20,6 +20,7 @@
 */
 #include "config.h"
 
+#include <ebos/eclmpiserializer.hh>
 
 #include <flow/flow_ebos_blackoil.hpp>
 
@@ -38,6 +39,7 @@
 #include <opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp>
 #include <opm/simulators/flow/FlowMainEbos.hpp>
 #include <opm/simulators/utils/moduleVersion.hpp>
+#include <opm/simulators/utils/ParallelEclipseState.hpp>
 #include <opm/models/utils/propertysystem.hh>
 #include <opm/models/utils/parametersystem.hh>
 #include <opm/simulators/flow/MissingFeatures.hpp>
@@ -348,13 +350,21 @@ int main(int argc, char** argv)
 
             Opm::FlowMainEbos<PreTypeTag>::printPRTHeader(outputCout);
 
-            deck.reset( new Opm::Deck( parser.parseFile(deckFilename , parseContext, errorGuard)));
-            Opm::MissingFeatures::checkKeywords(*deck, parseContext, errorGuard);
-            if ( outputCout )
-                Opm::checkDeck(*deck, parser, parseContext, errorGuard);
-
-            eclipseState.reset( new Opm::EclipseState(*deck));
+#ifdef HAVE_MPI
+            Opm::ParallelEclipseState* parState;
+#endif
             if (mpiRank == 0) {
+                deck.reset( new Opm::Deck( parser.parseFile(deckFilename , parseContext, errorGuard)));
+                Opm::MissingFeatures::checkKeywords(*deck, parseContext, errorGuard);
+                if ( outputCout )
+                    Opm::checkDeck(*deck, parser, parseContext, errorGuard);
+
+#ifdef HAVE_MPI
+                parState = new Opm::ParallelEclipseState(*deck);
+                eclipseState.reset(parState);
+#else
+                eclipseState.reset(new Opm::EclipseState(*deck);
+#endif
                 /*
                   For the time being initializing wells and groups from the
                   restart file is not possible, but work is underways and it is
@@ -382,9 +392,13 @@ int main(int argc, char** argv)
             else {
                 summaryConfig.reset(new Opm::SummaryConfig);
                 schedule.reset(new Opm::Schedule);
-                Opm::Mpi::receiveAndUnpack(*summaryConfig, Dune::MPIHelper::getCollectiveCommunication());
-                Opm::Mpi::receiveAndUnpack(*schedule, Dune::MPIHelper::getCollectiveCommunication());
+                parState = new Opm::ParallelEclipseState;
+                Opm::Mpi::receiveAndUnpack(*summaryConfig, mpiHelper.getCollectiveCommunication());
+                Opm::Mpi::receiveAndUnpack(*schedule, mpiHelper.getCollectiveCommunication());
+                eclipseState.reset(parState);
             }
+            Opm::EclMpiSerializer ser(mpiHelper.getCollectiveCommunication());
+            ser.broadcast(*parState);
 #endif
 
             Opm::checkConsistentArrayDimensions(*eclipseState, *schedule, parseContext, errorGuard);
diff --git a/opm/simulators/flow/FlowMainEbos.hpp b/opm/simulators/flow/FlowMainEbos.hpp
index 6898e1da9..30cd0ba17 100644
--- a/opm/simulators/flow/FlowMainEbos.hpp
+++ b/opm/simulators/flow/FlowMainEbos.hpp
@@ -450,11 +450,15 @@ namespace Opm
             // Run relperm diagnostics if we have more than one phase.
             if (FluidSystem::numActivePhases() > 1) {
                 RelpermDiagnostics diagnostic;
-                if (mpi_size_ > 1)
+                if (mpi_size_ > 1) {
                     this->grid().switchToGlobalView();
+                    static_cast<ParallelEclipseState&>(this->eclState()).switchToGlobalProps();
+                }
                 diagnostic.diagnosis(eclState(), deck(), this->grid());
-                if (mpi_size_ > 1)
+                if (mpi_size_ > 1) {
                     this->grid().switchToDistributedView();
+                    static_cast<ParallelEclipseState&>(this->eclState()).switchToDistributedProps();
+                }
             }
         }