Fix RFT State Collection for Distributed Wells

Commit 6295bd08c (PR #6238) restricted RFT data reporting to only
those wells that are owned by the current rank.  In so doing,
however, we failed to properly account for distributed wells.  Such
wells will typically have RFT contributions from multiple MPI ranks
in a parallel run and we did not allow for this situation.
Moreover, as the simulator by default will distribute "inactive"
wells--wells that are never opened, but which exist primarily to
collect RFT-style data--the problem becomes more pronounced for
these wells.

We solve the problem by splitting the RFTContainer's "wellQuery_"
into two responsibilities.  One for determining if a particular
named well is owned by the current rank and another for determining
if the well is intersected on the current rank.  The latter will be
true for all ranks that have local well connections for the named
well.  This, in turn, means that the constructor now takes two
predicate functions instead of the one we originally used.
Furthermore, member function RFTContainer::allocate() will create
internal data structures for all potential connections on all ranks
that share a well, but RFTContainer::addToWells() will still collect
the RFT data only on the rank that owns the well.

Finally, we implement the owning vs. intersecting distinction for
the black-oil output module through a new member function

    bool BlackoilWellModelGeneric<>::hasLocalCells(name) const

while the compositional output module retains the original
owning-is-intersecting behaviour.  We do not support distributed
wells in compositional runs at the time of writing.

CMakeLists_files.cmake

@@ -463,6 +463,7 @@ list (APPEND TEST_SOURCE_FILES
   tests/test_region_phase_pvaverage.cpp
   tests/test_relpermdiagnostics.cpp
   tests/test_RestartSerialization.cpp
+  tests/test_rftcontainer.cpp
   tests/test_RunningStatistics.cpp
   tests/test_rstconv.cpp
   tests/test_stoppedwells.cpp

opm/simulators/flow/GenericOutputBlackoilModule.cpp

@@ -157,8 +157,8 @@ GenericOutputBlackoilModule(const EclipseState& eclState,
     , enableMICP_(enableMICP)
     , flowsC_(schedule, summaryConfig)
     , rftC_(eclState_, schedule_,
-            [this](const std::string& wname)
-            { return this->isOwnedByCurrentRank(wname); })
+            [this](const std::string& wname) { return this->isOwnedByCurrentRank(wname); },
+            [this](const std::string& wname) { return this->isOnCurrentRank(wname); })
     , rst_conv_(std::move(globalCell), comm)
     , local_data_valid_(false)
 {

opm/simulators/flow/GenericOutputBlackoilModule.hpp

@@ -332,6 +332,7 @@ protected:
 
     virtual bool isDefunctParallelWell(const std::string& wname) const = 0;
     virtual bool isOwnedByCurrentRank(const std::string& wname) const = 0;
+    virtual bool isOnCurrentRank(const std::string& wname) const = 0;
 
     const EclipseState& eclState_;
     const Schedule& schedule_;

opm/simulators/flow/OutputBlackoilModule.hpp

@@ -658,6 +658,11 @@ private:
         return this->simulator_.problem().wellModel().isOwner(wname);
     }
 
+    bool isOnCurrentRank(const std::string& wname) const override
+    {
+        return this->simulator_.problem().wellModel().hasLocalCells(wname);
+    }
+
     void updateFluidInPlace_(const ElementContext& elemCtx, const unsigned dofIdx)
     {
         const auto& intQuants = elemCtx.intensiveQuantities(dofIdx, /*timeIdx=*/0);

opm/simulators/flow/OutputCompositionalModule.hpp

@@ -372,6 +372,14 @@ private:
         return ! this->isDefunctParallelWell(wname);
     }
 
+    bool isOnCurrentRank(const std::string& wname) const override
+    {
+        // Note: This statement is not correct for distributed wells and
+        // will need additional logic once those are supported for
+        // compositional flows.
+        return ! this->isDefunctParallelWell(wname);
+    }
+
     void createLocalRegion_(std::vector<int>& region)
     {
         std::size_t elemIdx = 0;

opm/simulators/flow/RFTContainer.cpp

@@ -88,7 +88,7 @@ addToWells(data::Wells& wellDatas,
 
     for (const auto& wname : this->schedule_[reportStepNum].well_order()) {
         // Don't bother with wells not on this process.
-        if (!wellQuery_(wname)) {
+        if (!wellIsOwnedByCurrentRank_(wname)) {
             continue;
         }
 
@@ -146,9 +146,8 @@ allocate(const std::size_t reportStepNum)
     }
 
     for (const auto& wname : schedule_[reportStepNum].well_order()) {
-
         // don't bother with wells not on this process
-        if (!wellQuery_(wname)) {
+        if (!wellOnCurrentRank_(wname)) {
             continue;
         }
 

opm/simulators/flow/RFTContainer.hpp

@@ -40,45 +40,156 @@ namespace data { class Wells; }
 class EclipseState;
 class Schedule;
 
+/// Collection of cell-level RFT data--i.e., pressures and saturations--in
+/// cells intersected by wells.  Will collect dynamic state values only for
+/// those cells for which RFT data has been requested through the Schedule's
+/// RFT configuration (member function ScheduleState::rft_config()).
+///
+/// \tparam FluidSystem Run's fluid system.  Needed, in particular, to infer
+/// the simulator's \c Scalar type and its active phases.
 template<class FluidSystem>
 class RFTContainer {
+    /// Simulator's floating-point element type.
     using Scalar = typename FluidSystem::Scalar;
+
+    /// Convenience type alias for a linear sequence of floating-point
+    /// values.
     using ScalarBuffer = std::vector<Scalar>;
 
+    /// Phase index for gas.
     static constexpr auto gasPhaseIdx = FluidSystem::gasPhaseIdx;
+
+    /// Phase index for oil.
     static constexpr auto oilPhaseIdx = FluidSystem::oilPhaseIdx;
+
+    /// Phase index for water.
     static constexpr auto waterPhaseIdx = FluidSystem::waterPhaseIdx;
 
 public:
+    /// Call-back function type for collecting cell level dynamic state
+    /// values.
     using AssignmentFunc = std::function<Scalar()>;
+
+    /// Call-back predicate type for inferring MPI characteristics for a
+    /// named well.  Common examples of such characteristics are whether or
+    /// not the current rank "owns" the well or if the well is intersected
+    /// on the current rank.
     using WellQueryFunc = std::function<bool(const std::string&)>;
 
+    /// Constructor.
+    ///
+    /// \param[in] eclState Run's static properties and configurations.
+    ///
+    /// \param[in] schedule Run's dynamic input objects such as its wells
+    /// and RFT configuration.
+    ///
+    /// \param[in] wellIsOwnedByCurrent Predicate for whether or not a
+    /// particular named well object is owned by the current rank.  The RFT
+    /// state data for a particular well will be collected on the owning
+    /// rank for that well.
+    ///
+    /// \param[in] wellOnCurrent Predicate for whether or not a particular
+    /// named well object is intersected on the current rank.  This function
+    /// will typically return \c true for those ranks that have local
+    /// connections for the well and false otherwise.
     RFTContainer(const EclipseState& eclState,
                  const Schedule& schedule,
-                 const WellQueryFunc& wellQuery)
+                 const WellQueryFunc& wellIsOwnedByCurrent,
+                 const WellQueryFunc& wellOnCurrent)
         : eclState_(eclState)
         , schedule_(schedule)
-        , wellQuery_(wellQuery)
+        , wellIsOwnedByCurrentRank_(wellIsOwnedByCurrent)
+        , wellOnCurrentRank_(wellOnCurrent)
     {}
 
-   void addToWells(data::Wells& wellDatas,
-                   const std::size_t reportStepNum,
-                   const Parallel::Communication& comm);
+    /// Export RFT cell level state data to requisite connections.
+    ///
+    /// Will populate the \code cell_pressure \endcode, the \code
+    /// cell_saturation_water \endcode, and the \code cell_saturation_gas
+    /// \endcode data members of the pertinent \code data::Connection
+    /// \endcode objects depending on the run's active phases.
+    ///
+    /// \param[in,out] wellDatas Well and connection level dynamic report
+    /// data at the current report step.  On exit, also contains all
+    /// relevant and available RFT state data for the wells owned by the
+    /// current rank.
+    ///
+    /// \param[in] reportStepNum Report step.  RFT data will be exported
+    /// only for those wells that request such values at this time.
+    ///
+    /// \param[in] comm MPI communication object.  Needed to collect RFT
+    /// state data from all ranks that share a well.
+    void addToWells(data::Wells& wellDatas,
+                    const std::size_t reportStepNum,
+                    const Parallel::Communication& comm);
 
+    /// Prepare internal data structures to collect RFT state.
+    ///
+    /// \param[in] reportStepNum Report step.  RFT data will be exported
+    /// only for those wells that request such values at this time.
     void allocate(const std::size_t reportStepNum);
 
-   void assign(const unsigned cartesianIndex,
+    /// Collect cell level RFT state into internal data structures.
+    ///
+    /// Does nothing if the cell is not among those for which RFT state is
+    /// requested.
+    ///
+    /// \param[in] cartesianIndex Linearised global cell ID.
+    ///
+    /// \param[in] oil Call-back for transferring cell level pressure values
+    /// in cell \p cartesianIndex from the simulator and into the container.
+    /// Will be invoked only if oil is active in the current run.
+    ///
+    /// \param[in] water Call-back for transferring cell level water
+    /// saturation values in cell \p cartesianIndex from the simulator and
+    /// into the container.  Will be invoked only if water is active in the
+    /// current run.
+    ///
+    /// \param[in] gas Call-back for transferring cell level gas saturation
+    /// values in cell \p cartesianIndex from the simulator and into the
+    /// container.  Will be invoked only if gas is active in the current
+    /// run.
+    void assign(const unsigned cartesianIndex,
                 const AssignmentFunc& oil,
                 const AssignmentFunc& water,
                 const AssignmentFunc& gas);
 
 private:
+    /// Run's static properties and configurations.
     const EclipseState& eclState_;
-    const Schedule& schedule_;
-    WellQueryFunc wellQuery_;
 
+    /// Run's dynamic input objects, e.g., wells and RFT configuration.
+    const Schedule& schedule_;
+
+    /// Predicate function for whether or not a particular named well is
+    /// owned by the current rank.
+    ///
+    /// RFT state data will be collected on the well's owning rank.
+    WellQueryFunc wellIsOwnedByCurrentRank_;
+
+    /// Predicate function for whether or not a particular named well is
+    /// intersected on the current rank.
+    ///
+    /// Needed to properly allocate the internal buffers.
+    WellQueryFunc wellOnCurrentRank_;
+
+    /// Cell level oil pressure values for all pertinent well connections.
+    ///
+    /// Keyed by the linearised global cell ID.  Will be populated only if
+    /// oil is active in the current run.
     std::map<std::size_t, Scalar> oilConnectionPressures_;
+
+    /// Cell level water saturation values for all pertinent well
+    /// connections.
+    ///
+    /// Keyed by the linearised global cell ID.  Will be populated only if
+    /// water is active in the current run.
     std::map<std::size_t, Scalar> waterConnectionSaturations_;
+
+    /// Cell level gas saturation values for all pertinent well connections.
+    ///
+    /// Keyed by the linearised global cell ID.  Will be populated only if
+    /// gas is active in the current run.
     std::map<std::size_t, Scalar> gasConnectionSaturations_;
 };
 

opm/simulators/wells/BlackoilWellModelGeneric.hpp

@@ -282,13 +282,14 @@ public:
 
     bool isOwner(const std::string& wname) const
     {
-        auto pwInfoPos = std::find_if(this->parallel_well_info_.begin(),
-                                      this->parallel_well_info_.end(),
-                                      [&wname](const auto& pwInfo)
-                                      { return pwInfo.name() == wname; });
+        return this->parallelWellSatisfies
+            (wname, [](const auto& pwInfo) { return pwInfo.isOwner(); });
+    }
 
-        return (pwInfoPos != this->parallel_well_info_.end())
-            && pwInfoPos->isOwner();
+    bool hasLocalCells(const std::string& wname) const
+    {
+        return this->parallelWellSatisfies
+            (wname, [](const auto& pwInfo) { return pwInfo.hasLocalCells(); });
     }
 
     const ConnectionIndexMap& connectionIndexMap(const std::size_t idx)
@@ -597,6 +598,31 @@ private:
     void updateEclWellsCTFFromAction(const int timeStepIdx,
                                      const SimulatorUpdate& sim_update);
 
+    /// Check if a well satisfies a particular MPI condition
+    ///
+    /// Common conditions in this context are whether or not a named well is
+    /// owned by the current rank or has connections on the current rank.
+    ///
+    /// \tparam Predicate Condition predicate function type.
+    ///
+    /// \param[in] wname Well name.
+    ///
+    /// \param[in] p Predicate function implementing the specific condition.
+    ///
+    /// \return Whether or not \p wname is a parallel well and, if so, if it
+    /// satisfies the condition \p p.
+    template <typename Predicate>
+    bool parallelWellSatisfies(const std::string& wname, Predicate&& p) const
+    {
+        auto pwInfoPos = std::find_if(this->parallel_well_info_.begin(),
+                                      this->parallel_well_info_.end(),
+                                      [&wname](const auto& pwInfo)
+                                      { return pwInfo.name() == wname; });
+
+        return (pwInfoPos != this->parallel_well_info_.end())
+            && p(*pwInfoPos);
+    }
+
     /// Run caller-defined code for each well owned by current rank
     ///
     /// 'const' version.

parallelUnitTests.cmake

@@ -96,6 +96,21 @@ opm_add_test(test_parallel_wbp_calculation_well_openconns
     2
 )
 
+foreach(NPROC 2 3 4)
+  opm_add_test(test_rftcontainer_np${NPROC}
+    EXE_NAME
+      test_rftcontainer
+    CONDITION
+      MPI_FOUND AND Boost_UNIT_TEST_FRAMEWORK_FOUND
+    DRIVER_ARGS
+      -n ${NPROC}
+      -b ${PROJECT_BINARY_DIR}
+    NO_COMPILE
+    PROCESSORS
+      ${NPROC}
+  )
+endforeach()
+
 foreach(NPROC 2 3 4)
   opm_add_test(test_parallel_region_phase_pvaverage_np${NPROC}
     EXE_NAME