Copy overlap data after local solve.

Also update intensive quantities in overlap afterwards.

ebos/FIBlackOilModel.hpp

@@ -58,6 +58,7 @@ public:
         : BlackOilModel<TypeTag>(simulator)
     {
     }
+
     void invalidateAndUpdateIntensiveQuantities(unsigned timeIdx) const
     {
         this->invalidateIntensiveQuantitiesCache(timeIdx);
@@ -78,17 +79,44 @@ public:
         }
     }
 
+    void invalidateAndUpdateIntensiveQuantitiesOverlap(unsigned timeIdx) const
+    {
+        // loop over all elements
+        ThreadedEntityIterator<GridView, /*codim=*/0> threadedElemIt(this->gridView_);
+#ifdef _OPENMP
+#pragma omp parallel
+#endif
+        {
+            ElementContext elemCtx(this->simulator_);
+            auto elemIt = threadedElemIt.beginParallel();
+            for (; !threadedElemIt.isFinished(elemIt); elemIt = threadedElemIt.increment()) {
+                if (elemIt->partitionType() != Dune::OverlapEntity) {
+                    continue;
+                }
+                const Element& elem = *elemIt;
+                elemCtx.updatePrimaryStencil(elem);
+                // Mark cache for this element as invalid.
+                const std::size_t numPrimaryDof = elemCtx.numPrimaryDof(timeIdx);
+                for (unsigned dofIdx = 0; dofIdx < numPrimaryDof; ++dofIdx) {
+                    const unsigned globalIndex = elemCtx.globalSpaceIndex(dofIdx, timeIdx);
+                    this->setIntensiveQuantitiesCacheEntryValidity(globalIndex, timeIdx, false);
+                }
+                // Update for this element.
+                elemCtx.updatePrimaryIntensiveQuantities(/*timeIdx=*/0);
+            }
+        }
+    }
+
     template <class GridSubDomain>
     void invalidateAndUpdateIntensiveQuantities(unsigned timeIdx, const GridSubDomain& gridSubDomain) const
     {
-        // loop over all elements...
+        // loop over all elements in the subdomain
         using GridViewType = decltype(gridSubDomain.view);
         ThreadedEntityIterator<GridViewType, /*codim=*/0> threadedElemIt(gridSubDomain.view);
 #ifdef _OPENMP
 #pragma omp parallel
 #endif
         {
-
             ElementContext elemCtx(this->simulator_);
             auto elemIt = threadedElemIt.beginParallel();
             for (; !threadedElemIt.isFinished(elemIt); elemIt = threadedElemIt.increment()) {

opm/simulators/flow/BlackoilModelEbosNldd.hpp

@@ -256,6 +256,20 @@ public:
             model_.ebosSimulator().model().invalidateAndUpdateIntensiveQuantities(/*timeIdx=*/0);
         }
 
+        // Communicate solutions:
+        // With multiple processes, this process' overlap (i.e. not
+        // owned) cells' solution values have been modified by local
+        // solves in the owning processes, and remain unchanged
+        // here. We must therefore receive the updated solution on the
+        // overlap cells and update their intensive quantities before
+        // we move on.
+        const auto& comm = model_.ebosSimulator().vanguard().grid().comm();
+        if (comm.size() > 1) {
+            const auto& ccomm = model_.ebosSimulator().model().newtonMethod().linearSolver().comm();
+            ccomm->copyOwnerToAll(solution, solution);
+            model_.ebosSimulator().model().invalidateAndUpdateIntensiveQuantitiesOverlap(/*timeIdx=*/0);
+        }
+
         // Finish with a Newton step.
         // Note that the "iteration + 100" is a simple way to avoid entering
         // "if (iteration == 0)" and similar blocks, and also makes it a little

opm/simulators/linalg/ISTLSolverEbos.hpp

@@ -352,6 +352,8 @@ std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
         /// \copydoc NewtonIterationBlackoilInterface::parallelInformation
         const std::any& parallelInformation() const { return parallelInformation_; }
 
+        const CommunicationType* comm() const { return comm_.get(); }
+
     protected:
 #if HAVE_MPI
         using Comm = Dune::OwnerOverlapCopyCommunication<int, int>;