changed: avoid usage of eclipseGrid on non-root processes setting up transmissibilities

obtain data from root process

ebos/eclcpgridvanguard.hh

@@ -151,7 +151,7 @@ public:
             if (grid_->size(0))
             {
                 globalTrans_.reset(new EclTransmissibility<TypeTag>(*this));
-                globalTrans_->update();
+                globalTrans_->update(false);
             }
 
             Dune::EdgeWeightMethod edgeWeightsMethod = this->edgeWeightsMethod();

ebos/eclproblem.hh

@@ -815,7 +815,7 @@ public:
             eclState.applyModifierDeck(miniDeck);
 
             // re-compute all quantities which may possibly be affected.
-            transmissibilities_.update();
+            transmissibilities_.update(true);
             referencePorosity_[1] = referencePorosity_[0];
             updateReferencePorosity_();
             updatePffDofData_();

ebos/ecltransmissibility.hh

@@ -117,22 +117,23 @@ public:
      * either but at least it seems to be much better.
      */
     void finishInit()
-    { update(); }
+    { update(true); }
 
 
     /*!
      * \brief Compute all transmissibilities
      *
+     * \param global If true, update is called on all processes
      * Also, this updates the "thermal half transmissibilities" if energy is enabled.
      */
-    void update()
+    void update(bool global)
     {
         const auto& gridView = vanguard_.gridView();
         const auto& cartMapper = vanguard_.cartesianIndexMapper();
         const auto& eclState = vanguard_.eclState();
-        const auto& eclGrid = eclState.getInputGrid();
         const auto& cartDims = cartMapper.cartesianDimensions();
         auto& transMult = eclState.getTransMult();
+        const auto& comm = vanguard_.gridView().comm();
         ElementMapper elemMapper(gridView, Dune::mcmgElementLayout());
 
         // get the ntg values, the ntg values are modified for the cells merged with minpv
@@ -148,17 +149,66 @@ public:
         for (unsigned dimIdx = 0; dimIdx < dimWorld; ++dimIdx)
             axisCentroids[dimIdx].resize(numElements);
 
+        std::vector<double> centroids;
+#if HAVE_MPI
+        size_t cells = vanguard_.grid().numCells();
+        if (global && comm.size() > 1) {
+            std::vector<size_t> sizes(comm.size());
+            if (comm.rank() == 0) {
+                const auto& eclGrid = eclState.getInputGrid();
+                comm.gather(&cells, sizes.data(), 1, 0);
+                for (int i = 1; i < comm.size(); ++i) {
+                    std::vector<int> cell_id(sizes[i]);
+                    MPI_Recv(cell_id.data(), sizes[i], MPI_INT,
+                             i, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+                    centroids.resize(dimWorld * sizes[i]);
+
+                    auto cIt = centroids.begin();
+                    for (int idx : cell_id) {
+                        const auto& centroid = eclGrid.getCellCenter(idx);
+                        for (const auto& it : centroid)
+                            *cIt++ = it;
+                    }
+                    MPI_Send(centroids.data(), dimWorld * sizes[i],
+                             MPI_DOUBLE, i, 0, MPI_COMM_WORLD);
+                }
+                centroids.clear();
+            } else {
+                comm.gather(&cells, sizes.data(), 1, 0);
+                std::vector<int> cell_ids;
+                cell_ids.reserve(cells);
+                auto elemIt = gridView.template begin</*codim=*/ 0>();
+                const auto& elemEndIt = gridView.template end</*codim=*/ 0>();
+                for (; elemIt != elemEndIt; ++elemIt) {
+                    const auto& elem = *elemIt;
+                    cell_ids.push_back(cartMapper.cartesianIndex(elemMapper.index(elem)));
+                }
+                MPI_Send(cell_ids.data(), cells, MPI_INT, 0, 0, MPI_COMM_WORLD);
+                centroids.resize(cells * dimWorld);
+                MPI_Recv(centroids.data(), dimWorld * cells, MPI_DOUBLE,
+                         0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+            }
+        }
+#endif
+
         auto elemIt = gridView.template begin</*codim=*/ 0>();
         const auto& elemEndIt = gridView.template end</*codim=*/ 0>();
-        for (; elemIt != elemEndIt; ++elemIt) {
+        size_t centroidIdx = 0;
+        for (; elemIt != elemEndIt; ++elemIt, ++centroidIdx) {
             const auto& elem = *elemIt;
             unsigned elemIdx = elemMapper.index(elem);
 
             // compute the axis specific "centroids" used for the transmissibilities. for
             // consistency with the flow simulator, we use the element centers as
             // computed by opm-parser's Opm::EclipseGrid class for all axes.
-            unsigned cartesianCellIdx = cartMapper.cartesianIndex(elemIdx);
+            const double* centroid;
-            const auto& centroid = eclGrid.getCellCenter(cartesianCellIdx);
+            if (vanguard_.gridView().comm().rank() == 0) {
+                const auto& eclGrid = eclState.getInputGrid();
+                unsigned cartesianCellIdx = cartMapper.cartesianIndex(elemIdx);
+                centroid = &eclGrid.getCellCenter(cartesianCellIdx)[0];
+            } else
+                centroid = &centroids[centroidIdx * dimWorld];
+
             for (unsigned axisIdx = 0; axisIdx < dimWorld; ++axisIdx)
                 for (unsigned dimIdx = 0; dimIdx < dimWorld; ++dimIdx)
                     axisCentroids[axisIdx][elemIdx][dimIdx] = centroid[dimIdx];
@@ -181,6 +231,18 @@ public:
             thermalHalfTransBoundary_.clear();
         }
 
+        // The MULTZ needs special case if the option is ALL
+        // Then the smallest multiplier is applied.
+        // Default is to apply the top and bottom multiplier
+        bool useSmallestMultiplier;
+        if (comm.rank() == 0) {
+            const auto& eclGrid = eclState.getInputGrid();
+            useSmallestMultiplier = eclGrid.getMultzOption() == Opm::PinchMode::ModeEnum::ALL;
+        }
+        if (global && comm.size() > 1) {
+            comm.broadcast(&useSmallestMultiplier, 1, 0);
+        }
+
         // compute the transmissibilities for all intersections
         elemIt = gridView.template begin</*codim=*/ 0>();
         for (; elemIt != elemEndIt; ++elemIt) {
@@ -330,10 +392,6 @@ public:
                 // apply the full face transmissibility multipliers
                 // for the inside ...
 
-                // The MULTZ needs special case if the option is ALL
-                // Then the smallest multiplier is applied.
-                // Default is to apply the top and bottom multiplier
-                bool useSmallestMultiplier = eclGrid.getMultzOption() == Opm::PinchMode::ModeEnum::ALL;
                 if (useSmallestMultiplier)
                     applyAllZMultipliers_(trans, insideFaceIdx, insideCartElemIdx, outsideCartElemIdx, transMult, cartDims);
                 else