From ac28bbfd1573de0555b9c627c9cc18c6135cdf14 Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Wed, 15 Jan 2020 14:36:52 +0100 Subject: [PATCH] changed: avoid usage of eclipseGrid on non-root processes setting up transmissibilities obtain data from root process --- ebos/eclcpgridvanguard.hh | 2 +- ebos/eclproblem.hh | 2 +- ebos/ecltransmissibility.hh | 78 ++++++++++++++++++++++++++++++++----- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/ebos/eclcpgridvanguard.hh b/ebos/eclcpgridvanguard.hh index 6e26ba43d..5ac5decdc 100644 --- a/ebos/eclcpgridvanguard.hh +++ b/ebos/eclcpgridvanguard.hh @@ -151,7 +151,7 @@ public: if (grid_->size(0)) { globalTrans_.reset(new EclTransmissibility(*this)); - globalTrans_->update(); + globalTrans_->update(false); } Dune::EdgeWeightMethod edgeWeightsMethod = this->edgeWeightsMethod(); diff --git a/ebos/eclproblem.hh b/ebos/eclproblem.hh index 9cb0f0235..a1c418f0f 100644 --- a/ebos/eclproblem.hh +++ b/ebos/eclproblem.hh @@ -815,7 +815,7 @@ public: eclState.applyModifierDeck(miniDeck); // re-compute all quantities which may possibly be affected. - transmissibilities_.update(); + transmissibilities_.update(true); referencePorosity_[1] = referencePorosity_[0]; updateReferencePorosity_(); updatePffDofData_(); diff --git a/ebos/ecltransmissibility.hh b/ebos/ecltransmissibility.hh index b8fba39a4..c9cd29c83 100644 --- a/ebos/ecltransmissibility.hh +++ b/ebos/ecltransmissibility.hh @@ -117,22 +117,23 @@ public: * either but at least it seems to be much better. */ void finishInit() - { update(); } + { update(true); } /*! * \brief Compute all transmissibilities * + * \param global If true, update is called on all processes * Also, this updates the "thermal half transmissibilities" if energy is enabled. */ - void update() + void update(bool global) { const auto& gridView = vanguard_.gridView(); const auto& cartMapper = vanguard_.cartesianIndexMapper(); const auto& eclState = vanguard_.eclState(); - const auto& eclGrid = eclState.getInputGrid(); const auto& cartDims = cartMapper.cartesianDimensions(); auto& transMult = eclState.getTransMult(); + const auto& comm = vanguard_.gridView().comm(); ElementMapper elemMapper(gridView, Dune::mcmgElementLayout()); // get the ntg values, the ntg values are modified for the cells merged with minpv @@ -148,17 +149,66 @@ public: for (unsigned dimIdx = 0; dimIdx < dimWorld; ++dimIdx) axisCentroids[dimIdx].resize(numElements); + std::vector centroids; +#if HAVE_MPI + size_t cells = vanguard_.grid().numCells(); + if (global && comm.size() > 1) { + std::vector sizes(comm.size()); + if (comm.rank() == 0) { + const auto& eclGrid = eclState.getInputGrid(); + comm.gather(&cells, sizes.data(), 1, 0); + for (int i = 1; i < comm.size(); ++i) { + std::vector cell_id(sizes[i]); + MPI_Recv(cell_id.data(), sizes[i], MPI_INT, + i, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + centroids.resize(dimWorld * sizes[i]); + + auto cIt = centroids.begin(); + for (int idx : cell_id) { + const auto& centroid = eclGrid.getCellCenter(idx); + for (const auto& it : centroid) + *cIt++ = it; + } + MPI_Send(centroids.data(), dimWorld * sizes[i], + MPI_DOUBLE, i, 0, MPI_COMM_WORLD); + } + centroids.clear(); + } else { + comm.gather(&cells, sizes.data(), 1, 0); + std::vector cell_ids; + cell_ids.reserve(cells); + auto elemIt = gridView.template begin(); + const auto& elemEndIt = gridView.template end(); + for (; elemIt != elemEndIt; ++elemIt) { + const auto& elem = *elemIt; + cell_ids.push_back(cartMapper.cartesianIndex(elemMapper.index(elem))); + } + MPI_Send(cell_ids.data(), cells, MPI_INT, 0, 0, MPI_COMM_WORLD); + centroids.resize(cells * dimWorld); + MPI_Recv(centroids.data(), dimWorld * cells, MPI_DOUBLE, + 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + } + } +#endif + auto elemIt = gridView.template begin(); const auto& elemEndIt = gridView.template end(); - for (; elemIt != elemEndIt; ++elemIt) { + size_t centroidIdx = 0; + for (; elemIt != elemEndIt; ++elemIt, ++centroidIdx) { const auto& elem = *elemIt; unsigned elemIdx = elemMapper.index(elem); // compute the axis specific "centroids" used for the transmissibilities. for // consistency with the flow simulator, we use the element centers as // computed by opm-parser's Opm::EclipseGrid class for all axes. - unsigned cartesianCellIdx = cartMapper.cartesianIndex(elemIdx); - const auto& centroid = eclGrid.getCellCenter(cartesianCellIdx); + const double* centroid; + if (vanguard_.gridView().comm().rank() == 0) { + const auto& eclGrid = eclState.getInputGrid(); + unsigned cartesianCellIdx = cartMapper.cartesianIndex(elemIdx); + centroid = &eclGrid.getCellCenter(cartesianCellIdx)[0]; + } else + centroid = ¢roids[centroidIdx * dimWorld]; + for (unsigned axisIdx = 0; axisIdx < dimWorld; ++axisIdx) for (unsigned dimIdx = 0; dimIdx < dimWorld; ++dimIdx) axisCentroids[axisIdx][elemIdx][dimIdx] = centroid[dimIdx]; @@ -181,6 +231,18 @@ public: thermalHalfTransBoundary_.clear(); } + // The MULTZ needs special case if the option is ALL + // Then the smallest multiplier is applied. + // Default is to apply the top and bottom multiplier + bool useSmallestMultiplier; + if (comm.rank() == 0) { + const auto& eclGrid = eclState.getInputGrid(); + useSmallestMultiplier = eclGrid.getMultzOption() == Opm::PinchMode::ModeEnum::ALL; + } + if (global && comm.size() > 1) { + comm.broadcast(&useSmallestMultiplier, 1, 0); + } + // compute the transmissibilities for all intersections elemIt = gridView.template begin(); for (; elemIt != elemEndIt; ++elemIt) { @@ -330,10 +392,6 @@ public: // apply the full face transmissibility multipliers // for the inside ... - // The MULTZ needs special case if the option is ALL - // Then the smallest multiplier is applied. - // Default is to apply the top and bottom multiplier - bool useSmallestMultiplier = eclGrid.getMultzOption() == Opm::PinchMode::ModeEnum::ALL; if (useSmallestMultiplier) applyAllZMultipliers_(trans, insideFaceIdx, insideCartElemIdx, outsideCartElemIdx, transMult, cartDims); else