OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

cosmetics

Browse Source
This commit is contained in:
Arne Morten Kvarving 2025-01-22 16:16:42 +01:00
parent e288a61c59
commit 76c947b04c
8 changed files with 149 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
opm/simulators/aquifers/BlackoilAquiferModel_impl.hpp
View File

@ -139,9 +139,10 @@ BlackoilAquiferModel<TypeTag>::endTimeStep()
for (auto& aquifer : this->aquifers) { for (auto& aquifer : this->aquifers) {
aquifer->endTimeStep(); aquifer->endTimeStep();
NumAq* num = dynamic_cast<NumAq*>(aquifer.get()); NumAq* num = dynamic_cast<NumAq*>(aquifer.get());
if (num) if (num) {
this->simulator_.vanguard().grid().comm().barrier(); this->simulator_.vanguard().grid().comm().barrier();
} }
}
} }
template <typename TypeTag> template <typename TypeTag>

41
opm/simulators/flow/GenericThresholdPressure_impl.hpp
View File

@ -73,10 +73,11 @@ thresholdPressure(int elem1Idx, int elem2Idx) const
int fault1Idx = lookUpCartesianData_(elem1Idx, cartElemFaultIdx_); int fault1Idx = lookUpCartesianData_(elem1Idx, cartElemFaultIdx_);
int fault2Idx = lookUpCartesianData_(elem2Idx, cartElemFaultIdx_); int fault2Idx = lookUpCartesianData_(elem2Idx, cartElemFaultIdx_);
if (fault1Idx != -1 && fault1Idx == fault2Idx) if (fault1Idx != -1 && fault1Idx == fault2Idx) {
// inside a fault there's no threshold pressure, even accross EQUIL // inside a fault there's no threshold pressure, even accross EQUIL
// regions. // regions.
return 0.0; return 0.0;
}
if (fault1Idx != fault2Idx) { if (fault1Idx != fault2Idx) {
// TODO: which value if a cell is part of multiple faults? we take // TODO: which value if a cell is part of multiple faults? we take
// the maximum here. // the maximum here.
@ -90,8 +91,9 @@ thresholdPressure(int elem1Idx, int elem2Idx) const
auto equilRegion1Idx = elemEquilRegion_[elem1Idx]; auto equilRegion1Idx = elemEquilRegion_[elem1Idx];
auto equilRegion2Idx = elemEquilRegion_[elem2Idx]; auto equilRegion2Idx = elemEquilRegion_[elem2Idx];
if (equilRegion1Idx == equilRegion2Idx) if (equilRegion1Idx == equilRegion2Idx) {
return 0.0; return 0.0;
}
return thpres_[equilRegion1Idx*numEquilRegions_ + equilRegion2Idx]; return thpres_[equilRegion1Idx*numEquilRegions_ + equilRegion2Idx];
} }
@ -103,8 +105,9 @@ finishInit()
const auto& simConfig = eclState_.getSimulationConfig(); const auto& simConfig = eclState_.getSimulationConfig();
enableThresholdPressure_ = simConfig.useThresholdPressure(); enableThresholdPressure_ = simConfig.useThresholdPressure();
if (!enableThresholdPressure_) if (!enableThresholdPressure_) {
return; return;
}
numEquilRegions_ = eclState_.getTableManager().getEqldims().getNumEquilRegions(); numEquilRegions_ = eclState_.getTableManager().getEqldims().getNumEquilRegions();
const decltype(numEquilRegions_) maxRegions = const decltype(numEquilRegions_) maxRegions =
@ -129,7 +132,8 @@ finishInit()
} }
// internalize the data specified using the EQLNUM keyword // internalize the data specified using the EQLNUM keyword
elemEquilRegion_ = lookUpData_.template assignFieldPropsIntOnLeaf<short unsigned int>(eclState_.fieldProps(), elemEquilRegion_ = lookUpData_.
template assignFieldPropsIntOnLeaf<short unsigned int>(eclState_.fieldProps(),
"EQLNUM", true); "EQLNUM", true);
/* /*
@ -137,8 +141,9 @@ finishInit()
and already properly initialized with numerical values from the restart. and already properly initialized with numerical values from the restart.
Done using GenericThresholdPressure::setFromRestart() in EclWriter::beginRestart(). Done using GenericThresholdPressure::setFromRestart() in EclWriter::beginRestart().
*/ */
if (simConfig.getThresholdPressure().restart()) if (simConfig.getThresholdPressure().restart()) {
return; return;
}
// allocate the array which specifies the threshold pressures // allocate the array which specifies the threshold pressures
thpres_.resize(numEquilRegions_*numEquilRegions_, 0.0); thpres_.resize(numEquilRegions_*numEquilRegions_, 0.0);
@ -156,10 +161,12 @@ applyExplicitThresholdPressures_()
// intersection in the grid // intersection in the grid
for (const auto& elem : elements(gridView_, Dune::Partitions::interior)) { for (const auto& elem : elements(gridView_, Dune::Partitions::interior)) {
for (const auto& intersection : intersections(gridView_, elem)) { for (const auto& intersection : intersections(gridView_, elem)) {
if (intersection.boundary()) if (intersection.boundary()) {
continue; // ignore boundary intersections for now (TODO?) continue; // ignore boundary intersections for now (TODO?)
else if (!intersection.neighbor()) //processor boundary but not domain boundary }
else if (!intersection.neighbor()) { // processor boundary but not domain boundary
continue; continue;
}
const auto& inside = intersection.inside(); const auto& inside = intersection.inside();
const auto& outside = intersection.outside(); const auto& outside = intersection.outside();
@ -189,8 +196,9 @@ applyExplicitThresholdPressures_()
} }
// apply threshold pressures across faults // apply threshold pressures across faults
if (thpres.ftSize() > 0) if (thpres.ftSize() > 0) {
configureThpresft_(); configureThpresft_();
}
} }
template<class Grid, class GridView, class ElementMapper, class Scalar> template<class Grid, class GridView, class ElementMapper, class Scalar>
@ -207,15 +215,17 @@ configureThpresft_()
int numCartesianElem = eclState_.getInputGrid().getCartesianSize(); int numCartesianElem = eclState_.getInputGrid().getCartesianSize();
thpresftValues_.resize(numFaults, -1.0); thpresftValues_.resize(numFaults, -1.0);
cartElemFaultIdx_.resize(numCartesianElem, -1); cartElemFaultIdx_.resize(numCartesianElem, -1);
for (std::size_t faultIdx = 0; faultIdx < faults.size(); faultIdx++) { for (std::size_t faultIdx = 0; faultIdx < faults.size(); ++faultIdx) {
auto& fault = faults.getFault(faultIdx); auto& fault = faults.getFault(faultIdx);
thpresftValues_[faultIdx] = thpres.getThresholdPressureFault(faultIdx); thpresftValues_[faultIdx] = thpres.getThresholdPressureFault(faultIdx);
for (const FaultFace& face : fault) for (const FaultFace& face : fault) {
// "face" is a misnomer because the object describes a set of cell // "face" is a misnomer because the object describes a set of cell
// indices, but we go with the conventions of the parser here... // indices, but we go with the conventions of the parser here...
for (std::size_t cartElemIdx : face) for (std::size_t cartElemIdx : face) {
cartElemFaultIdx_[cartElemIdx] = faultIdx; cartElemFaultIdx_[cartElemIdx] = faultIdx;
} }
}
}
} }
template<class Grid, class GridView, class ElementMapper, class Scalar> template<class Grid, class GridView, class ElementMapper, class Scalar>
@ -223,8 +233,9 @@ std::vector<Scalar>
GenericThresholdPressure<Grid,GridView,ElementMapper,Scalar>:: GenericThresholdPressure<Grid,GridView,ElementMapper,Scalar>::
getRestartVector() const getRestartVector() const
{ {
if (!enableThresholdPressure_) if (!enableThresholdPressure_) {
return {}; return {};
}
return this->thpres_; return this->thpres_;
} }
@ -243,8 +254,9 @@ void
GenericThresholdPressure<Grid,GridView,ElementMapper,Scalar>:: GenericThresholdPressure<Grid,GridView,ElementMapper,Scalar>::
logPressures() logPressures()
{ {
if (!enableThresholdPressure_) if (!enableThresholdPressure_) {
return; return;
}
auto lineFormat = [this](unsigned i, unsigned j, double val) auto lineFormat = [this](unsigned i, unsigned j, double val)
{ {
@ -274,7 +286,8 @@ logPressures()
if (thpres.hasRegionBarrier(i, j)) { if (thpres.hasRegionBarrier(i, j)) {
if (thpres.hasThresholdPressure(i, j)) { if (thpres.hasThresholdPressure(i, j)) {
str += lineFormat(i, j, thpres.getThresholdPressure(j, i)); str += lineFormat(i, j, thpres.getThresholdPressure(j, i));
} else { }
else {
std::size_t idx = (j - 1) * numEquilRegions_ + (i - 1); std::size_t idx = (j - 1) * numEquilRegions_ + (i - 1);
str += lineFormat(i, j, this->thpresDefault_[idx]); str += lineFormat(i, j, this->thpresDefault_[idx]);
} }

2
opm/simulators/flow/TracerModel.hpp
View File

@ -892,7 +892,7 @@ protected:
TracerBatch(int phaseIdx = 0) : phaseIdx_(phaseIdx) {} TracerBatch(int phaseIdx = 0) : phaseIdx_(phaseIdx) {}
int numTracer() const {return idx_.size(); } int numTracer() const { return idx_.size(); }
void addTracer(const int idx, const TV & concentration) void addTracer(const int idx, const TV & concentration)
{ {

74
opm/simulators/linalg/gpubridge/Reorder.cpp
View File

@ -23,28 +23,28 @@
#include <opm/simulators/linalg/gpubridge/Reorder.hpp> #include <opm/simulators/linalg/gpubridge/Reorder.hpp>
#include <vector>
#include <cassert> #include <cassert>
#include <vector>
namespace Opm::Accelerator {
namespace Opm
{
namespace Accelerator
{
/* Check is operations on a node in the matrix can be started /* Check is operations on a node in the matrix can be started
* A node can only be started if all nodes that it depends on during sequential execution have already completed.*/ * A node can only be started if all nodes that it depends on during sequential
* execution have already completed.*/
bool canBeStarted(const int rowIndex, const int *rowPointers, const int *colIndices, const std::vector<bool>& doneRows) { bool canBeStarted(const int rowIndex,
const int* rowPointers,
const int* colIndices,
const std::vector<bool>& doneRows)
{
bool canStart = !doneRows[rowIndex]; bool canStart = !doneRows[rowIndex];
int i, thisDependency;
if (canStart) { if (canStart) {
for (i = rowPointers[rowIndex]; i < rowPointers[rowIndex + 1]; i++) { for (int i = rowPointers[rowIndex]; i < rowPointers[rowIndex + 1]; ++i) {
thisDependency = colIndices[i]; int thisDependency = colIndices[i];
// Only dependencies on rows that should execute before the current one are relevant // Only dependencies on rows that should execute before the current one are relevant
if (thisDependency >= rowIndex) if (thisDependency >= rowIndex) {
break; break;
}
// Check if dependency has been resolved // Check if dependency has been resolved
if (!doneRows[thisDependency]) { if (!doneRows[thisDependency]) {
return false; return false;
@ -55,14 +55,23 @@ bool canBeStarted(const int rowIndex, const int *rowPointers, const int *colIndi
} }
/* /*
* The level scheduling of a non-symmetric, blocked matrix requires access to a CSC encoding and a CSR encoding of the sparsity pattern of the input matrix. * The level scheduling of a non-symmetric, blocked matrix requires access to a CSC
* encoding and a CSR encoding of the sparsity pattern of the input matrix.
* This function is based on a standard level scheduling algorithm, like the one described in: * This function is based on a standard level scheduling algorithm, like the one described in:
* "Iterative methods for Sparse Linear Systems" by Yousef Saad in section 11.6.3 * "Iterative methods for Sparse Linear Systems" by Yousef Saad in section 11.6.3
*/ */
void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices, int *CSCColPointers, int Nb, int *numColors, int *toOrder, int* fromOrder, std::vector<int>& rowsPerColor) { void findLevelScheduling(int* CSRColIndices,
int activeRowIndex = 0, colorEnd, nextActiveRowIndex = 0; int* CSRRowPointers,
int thisRow; int* CSCRowIndices,
int* CSCColPointers,
int Nb,
int* numColors,
int* toOrder,
int* fromOrder,
std::vector<int>& rowsPerColor)
{
int activeRowIndex = 0, nextActiveRowIndex = 0;
std::vector<bool> doneRows(Nb, false); std::vector<bool> doneRows(Nb, false);
std::vector <int> rowsToStart; std::vector <int> rowsToStart;
@ -70,23 +79,27 @@ void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowInd
assert(rowsPerColor.empty()); assert(rowsPerColor.empty());
// find starting rows: rows that are independent from all rows that come before them. // find starting rows: rows that are independent from all rows that come before them.
for (thisRow = 0; thisRow < Nb; thisRow++) { int thisRow;
for (thisRow = 0; thisRow < Nb; ++thisRow) {
if (canBeStarted(thisRow, CSCColPointers, CSCRowIndices, doneRows)) { if (canBeStarted(thisRow, CSCColPointers, CSCRowIndices, doneRows)) {
fromOrder[nextActiveRowIndex] = thisRow; fromOrder[nextActiveRowIndex] = thisRow;
toOrder[thisRow] = nextActiveRowIndex; toOrder[thisRow] = nextActiveRowIndex;
nextActiveRowIndex++; ++nextActiveRowIndex;
} }
} }
// 'do' compute on all active rows // 'do' compute on all active rows
for (colorEnd = 0; colorEnd < nextActiveRowIndex; colorEnd++) { int colorEnd;
for (colorEnd = 0; colorEnd < nextActiveRowIndex; ++colorEnd) {
doneRows[fromOrder[colorEnd]] = true; doneRows[fromOrder[colorEnd]] = true;
} }
rowsPerColor.emplace_back(nextActiveRowIndex - activeRowIndex); rowsPerColor.emplace_back(nextActiveRowIndex - activeRowIndex);
while (colorEnd < Nb) { while (colorEnd < Nb) {
// Go over all rows active from the last color, and check which of their neighbours can be activated this color // Go over all rows active from the last color, and check which of
for (; activeRowIndex < colorEnd; activeRowIndex++) { // their neighbours can be activated this color
for (; activeRowIndex < colorEnd; ++activeRowIndex) {
thisRow = fromOrder[activeRowIndex]; thisRow = fromOrder[activeRowIndex];
for (int i = CSCColPointers[thisRow]; i < CSCColPointers[thisRow + 1]; i++) { for (int i = CSCColPointers[thisRow]; i < CSCColPointers[thisRow + 1]; i++) {
@ -104,7 +117,7 @@ void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowInd
doneRows[thisRow] = true; doneRows[thisRow] = true;
fromOrder[nextActiveRowIndex] = thisRow; fromOrder[nextActiveRowIndex] = thisRow;
toOrder[thisRow] = nextActiveRowIndex; toOrder[thisRow] = nextActiveRowIndex;
nextActiveRowIndex++; ++nextActiveRowIndex;
} }
} }
rowsToStart.clear(); rowsToStart.clear();
@ -115,10 +128,13 @@ void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowInd
*numColors = rowsPerColor.size(); *numColors = rowsPerColor.size();
} }
// based on the scipy package from python, scipy/sparse/sparsetools/csr.h on github // based on the scipy package from python, scipy/sparse/sparsetools/csr.h on github
void csrPatternToCsc(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices, int *CSCColPointers, int Nb) { void csrPatternToCsc(int* CSRColIndices,
int* CSRRowPointers,
int* CSCRowIndices,
int* CSCColPointers,
int Nb)
{
int nnz = CSRRowPointers[Nb]; int nnz = CSRRowPointers[Nb];
// compute number of nnzs per column // compute number of nnzs per column
@ -141,7 +157,7 @@ void csrPatternToCsc(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices
int col = CSRColIndices[j]; int col = CSRColIndices[j];
int dest = CSCColPointers[col]; int dest = CSCColPointers[col];
CSCRowIndices[dest] = row; CSCRowIndices[dest] = row;
CSCColPointers[col]++; ++CSCColPointers[col];
} }
} }
@ -152,6 +168,4 @@ void csrPatternToCsc(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices
} }
} }
} // namespace Opm::Accelerator
} // namespace Accelerator
} // namespace Opm

29
opm/simulators/linalg/gpubridge/Reorder.hpp
View File

@ -22,10 +22,7 @@
#include <vector> #include <vector>
namespace Opm namespace Opm::Accelerator {
{
namespace Accelerator
{
/// Determine whether all rows that a certain row depends on are done already /// Determine whether all rows that a certain row depends on are done already
/// \param[in] rowIndex index of the row that needs to be checked for /// \param[in] rowIndex index of the row that needs to be checked for
@ -33,7 +30,10 @@ namespace Accelerator
/// \param[in] colIndices column indices of the matrix that the row is in /// \param[in] colIndices column indices of the matrix that the row is in
/// \param[in] doneRows array that for each row lists whether it is done or not /// \param[in] doneRows array that for each row lists whether it is done or not
/// \return true iff all dependencies are done and if the result itself was not done yet /// \return true iff all dependencies are done and if the result itself was not done yet
bool canBeStarted(const int rowIndex, const int *rowPointers, const int *colIndices, const std::vector<bool>& doneRows); bool canBeStarted(const int rowIndex,
const int* rowPointers,
const int* colIndices,
const std::vector<bool>& doneRows);
/// Find a level scheduling reordering for an input matrix /// Find a level scheduling reordering for an input matrix
/// The toOrder and fromOrder arrays must be allocated already /// The toOrder and fromOrder arrays must be allocated already
@ -46,7 +46,15 @@ bool canBeStarted(const int rowIndex, const int *rowPointers, const int *colIn
/// \param[out] toOrder the reorder pattern that was found, which lists for each index in the original order, to which index in the new order it should be moved /// \param[out] toOrder the reorder pattern that was found, which lists for each index in the original order, to which index in the new order it should be moved
/// \param[out] fromOrder the reorder pattern that was found, which lists for each index in the new order, from which index in the original order it was moved /// \param[out] fromOrder the reorder pattern that was found, which lists for each index in the new order, from which index in the original order it was moved
/// \param[out] rowsPerColor for each color, an array of all rowIndices in that color, this function uses emplace_back() to fill /// \param[out] rowsPerColor for each color, an array of all rowIndices in that color, this function uses emplace_back() to fill
void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices, int *CSCColPointers, int Nb, int *numColors, int *toOrder, int* fromOrder, std::vector<int>& rowsPerColor); void findLevelScheduling(int* CSRColIndices,
int* CSRRowPointers,
int* CSCRowIndices,
int* CSCColPointers,
int Nb,
int* numColors,
int* toOrder,
int* fromOrder,
std::vector<int>& rowsPerColor);
/// Convert a sparsity pattern stored in the CSR format to the CSC format /// Convert a sparsity pattern stored in the CSR format to the CSC format
/// CSCRowIndices and CSCColPointers arrays must be allocated already /// CSCRowIndices and CSCColPointers arrays must be allocated already
@ -56,9 +64,12 @@ void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowInd
/// \param[inout] CSCRowIndices row indices of the result CSC representation of the pattern /// \param[inout] CSCRowIndices row indices of the result CSC representation of the pattern
/// \param[inout] CSCColPointers column pointers of the result CSC representation of the pattern /// \param[inout] CSCColPointers column pointers of the result CSC representation of the pattern
/// \param[in] Nb number of blockrows in the matrix /// \param[in] Nb number of blockrows in the matrix
void csrPatternToCsc(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices, int *CSCColPointers, int Nb); void csrPatternToCsc(int* CSRColIndices,
int* CSRRowPointers,
int* CSCRowIndices,
int* CSCColPointers,
int Nb);
} // namespace Accelerator } // namespace Opm::Accelerator
} // namespace Opm
#endif #endif

3
opm/simulators/linalg/gpubridge/opencl/openclBILU0.cpp
View File

@ -198,8 +198,9 @@ create_preconditioner(BlockedMatrix<Scalar>* mat, BlockedMatrix<Scalar>* jacMat)
bool use_multithreading = true; bool use_multithreading = true;
#if HAVE_OPENMP #if HAVE_OPENMP
if (omp_get_max_threads() == 1) if (omp_get_max_threads() == 1) {
use_multithreading = false; use_multithreading = false;
}
#endif #endif
if (jacMat && use_multithreading) { if (jacMat && use_multithreading) {

65
opm/simulators/linalg/gpuistl/GpuOwnerOverlapCopy.hpp
View File

@ -24,8 +24,8 @@
#include <opm/simulators/linalg/gpuistl/GpuVector.hpp> #include <opm/simulators/linalg/gpuistl/GpuVector.hpp>
#include <vector> #include <vector>
namespace Opm::gpuistl namespace Opm::gpuistl {
{
/** /**
* @brief GPUSender is a wrapper class for classes which will implement copOwnerToAll * @brief GPUSender is a wrapper class for classes which will implement copOwnerToAll
* This is implemented with the intention of creating communicators with generic GPUSender * This is implemented with the intention of creating communicators with generic GPUSender
@ -122,7 +122,8 @@ public:
{ {
} }
void copyOwnerToAll(const X& source, X& dest) const override { void copyOwnerToAll(const X& source, X& dest) const override
{
// TODO: [perf] Can we reduce copying from the GPU here? // TODO: [perf] Can we reduce copying from the GPU here?
// TODO: [perf] Maybe create a global buffer instead? // TODO: [perf] Maybe create a global buffer instead?
auto sourceAsDuneVector = source.template asDuneBlockVector<block_size>(); auto sourceAsDuneVector = source.template asDuneBlockVector<block_size>();
@ -179,7 +180,6 @@ public:
void copyOwnerToAll(const X& source, X& dest) const override void copyOwnerToAll(const X& source, X& dest) const override
{ {
OPM_ERROR_IF(&source != &dest, "The provided GpuVectors' address did not match"); // In this context, source == dest!!! OPM_ERROR_IF(&source != &dest, "The provided GpuVectors' address did not match"); // In this context, source == dest!!!
std::call_once(this->m_initializedIndices, [&]() { initIndexSet(); }); std::call_once(this->m_initializedIndices, [&]() { initIndexSet(); });
@ -198,9 +198,9 @@ public:
{ {
size_t i = 0; size_t i = 0;
for(const_iterator info = m_messageInformation.begin(); info != end; ++info, ++i) { for (const_iterator info = m_messageInformation.begin(); info != end; ++info, ++i) {
processMap[i]=info->first; processMap[i]=info->first;
if(info->second.second.m_size) { if (info->second.second.m_size) {
MPI_Irecv(m_GPURecvBuf->data()+info->second.second.m_start, MPI_Irecv(m_GPURecvBuf->data()+info->second.second.m_start,
detail::to_int(info->second.second.m_size), detail::to_int(info->second.second.m_size),
MPI_BYTE, MPI_BYTE,
@ -209,16 +209,17 @@ public:
this->m_cpuOwnerOverlapCopy.communicator(), this->m_cpuOwnerOverlapCopy.communicator(),
&recvRequests[i]); &recvRequests[i]);
numberOfRealRecvRequests += 1; numberOfRealRecvRequests += 1;
} else { }
recvRequests[i]=MPI_REQUEST_NULL; else {
recvRequests[i] = MPI_REQUEST_NULL;
} }
} }
} }
{ {
size_t i = 0; size_t i = 0;
for(const_iterator info = m_messageInformation.begin(); info != end; ++info, ++i) { for (const_iterator info = m_messageInformation.begin(); info != end; ++info, ++i) {
if(info->second.first.m_size) { if (info->second.first.m_size) {
MPI_Issend(m_GPUSendBuf->data()+info->second.first.m_start, MPI_Issend(m_GPUSendBuf->data()+info->second.first.m_start,
detail::to_int(info->second.first.m_size), detail::to_int(info->second.first.m_size),
MPI_BYTE, MPI_BYTE,
@ -227,24 +228,28 @@ public:
this->m_cpuOwnerOverlapCopy.communicator(), this->m_cpuOwnerOverlapCopy.communicator(),
&sendRequests[i]); &sendRequests[i]);
} else { } else {
sendRequests[i]=MPI_REQUEST_NULL; sendRequests[i] = MPI_REQUEST_NULL;
} }
} }
} }
int finished = MPI_UNDEFINED; int finished = MPI_UNDEFINED;
MPI_Status status; MPI_Status status;
for(size_t i = 0; i < numberOfRealRecvRequests; i++) { for (size_t i = 0; i < numberOfRealRecvRequests; i++) {
status.MPI_ERROR=MPI_SUCCESS; status.MPI_ERROR=MPI_SUCCESS;
MPI_Waitany(m_messageInformation.size(), recvRequests.data(), &finished, &status); MPI_Waitany(m_messageInformation.size(), recvRequests.data(), &finished, &status);
if(status.MPI_ERROR!=MPI_SUCCESS) { if (status.MPI_ERROR!=MPI_SUCCESS) {
OPM_THROW(std::runtime_error, fmt::format("MPI_Error occurred while rank {} received a message from rank {}", rank, processMap[finished])); OPM_THROW(std::runtime_error,
fmt::format("MPI_Error occurred while rank {} received a message from rank {}",
rank, processMap[finished]));
} }
} }
MPI_Status recvStatus; MPI_Status recvStatus;
for(size_t i = 0; i < m_messageInformation.size(); i++) { for (size_t i = 0; i < m_messageInformation.size(); i++) {
if(MPI_SUCCESS!=MPI_Wait(&sendRequests[i], &recvStatus)) { if (MPI_SUCCESS != MPI_Wait(&sendRequests[i], &recvStatus)) {
OPM_THROW(std::runtime_error, fmt::format("MPI_Error occurred while rank {} sent a message from rank {}", rank, processMap[finished])); OPM_THROW(std::runtime_error,
fmt::format("MPI_Error occurred while rank {} sent a message from rank {}",
rank, processMap[finished]));
} }
} }
// ...End of MPI stuff // ...End of MPI stuff
@ -279,20 +284,21 @@ private:
std::vector<int> commpairIndicesCopyOnCPU; std::vector<int> commpairIndicesCopyOnCPU;
std::vector<int> commpairIndicesOwnerCPU; std::vector<int> commpairIndicesOwnerCPU;
for(auto process : ri) { for (auto process : ri) {
m_im[process.first] = std::pair(std::vector<int>(), std::vector<int>()); m_im[process.first] = std::pair(std::vector<int>(), std::vector<int>());
for(int send = 0; send < 2; ++send) { for (int send = 0; send < 2; ++send) {
auto remoteEnd = send ? process.second.first->end() auto remoteEnd = send ? process.second.first->end()
: process.second.second->end(); : process.second.second->end();
auto remote = send ? process.second.first->begin() auto remote = send ? process.second.first->begin()
: process.second.second->begin(); : process.second.second->begin();
while(remote != remoteEnd) { while (remote != remoteEnd) {
if (send ? (remote->localIndexPair().local().attribute() == 1) if (send ? (remote->localIndexPair().local().attribute() == 1)
: (remote->attribute() == 1)) { : (remote->attribute() == 1)) {
if (send) { if (send) {
m_im[process.first].first.push_back(remote->localIndexPair().local().local()); m_im[process.first].first.push_back(remote->localIndexPair().local().local());
} else { }
else {
m_im[process.first].second.push_back(remote->localIndexPair().local().local()); m_im[process.first].second.push_back(remote->localIndexPair().local().local());
} }
} }
@ -317,13 +323,13 @@ private:
recvBufIdx * block_size, recvBufIdx * block_size,
noRecv * block_size * sizeof(field_type))))); noRecv * block_size * sizeof(field_type)))));
for(int x = 0; x < noSend; x++) { for (int x = 0; x < noSend; x++) {
for(int bs = 0; bs < block_size; bs++) { for (int bs = 0; bs < block_size; bs++) {
commpairIndicesOwnerCPU.push_back(it->second.first[x] * block_size + bs); commpairIndicesOwnerCPU.push_back(it->second.first[x] * block_size + bs);
} }
} }
for(int x = 0; x < noRecv; x++) { for (int x = 0; x < noRecv; x++) {
for(int bs = 0; bs < block_size; bs++) { for (int bs = 0; bs < block_size; bs++) {
commpairIndicesCopyOnCPU.push_back(it->second.second[x] * block_size + bs); commpairIndicesCopyOnCPU.push_back(it->second.second[x] * block_size + bs);
} }
} }
@ -385,9 +391,12 @@ class GpuOwnerOverlapCopy
public: public:
using X = GpuVector<field_type>; using X = GpuVector<field_type>;
explicit GpuOwnerOverlapCopy(std::shared_ptr<GPUSender<field_type, OwnerOverlapCopyCommunicationType>> sender) : m_sender(sender){} explicit GpuOwnerOverlapCopy(std::shared_ptr<GPUSender<field_type, OwnerOverlapCopyCommunicationType>> sender)
: m_sender(sender)
{}
void copyOwnerToAll(const X& source, X& dest) const { void copyOwnerToAll(const X& source, X& dest) const
{
m_sender->copyOwnerToAll(source, dest); m_sender->copyOwnerToAll(source, dest);
} }
@ -409,5 +418,7 @@ public:
private: private:
std::shared_ptr<GPUSender<field_type, OwnerOverlapCopyCommunicationType>> m_sender; std::shared_ptr<GPUSender<field_type, OwnerOverlapCopyCommunicationType>> m_sender;
}; };
} // namespace Opm::gpuistl } // namespace Opm::gpuistl
#endif #endif

14
opm/simulators/wells/BlackoilWellModel_impl.hpp
View File

@ -2011,7 +2011,10 @@ namespace Opm {
const int iterationIdx, const int iterationIdx,
DeferredLogger& deferred_logger) DeferredLogger& deferred_logger)
{ {
this->updateAndCommunicateGroupData(reportStepIdx, iterationIdx, param_.nupcol_group_rate_tolerance_, deferred_logger); this->updateAndCommunicateGroupData(reportStepIdx,
iterationIdx,
param_.nupcol_group_rate_tolerance_,
deferred_logger);
// updateWellStateWithTarget might throw for multisegment wells hence we // updateWellStateWithTarget might throw for multisegment wells hence we
// have a parallel try catch here to thrown on all processes. // have a parallel try catch here to thrown on all processes.
@ -2020,14 +2023,19 @@ namespace Opm {
for (const auto& well : well_container_) { for (const auto& well : well_container_) {
// We only want to update wells under group-control here // We only want to update wells under group-control here
auto& ws = this->wellState().well(well->indexOfWell()); auto& ws = this->wellState().well(well->indexOfWell());
if (ws.production_cmode == Well::ProducerCMode::GRUP || ws.injection_cmode == Well::InjectorCMode::GRUP) { if (ws.production_cmode == Well::ProducerCMode::GRUP ||
ws.injection_cmode == Well::InjectorCMode::GRUP)
{
well->updateWellStateWithTarget(simulator_, this->groupState(), well->updateWellStateWithTarget(simulator_, this->groupState(),
this->wellState(), deferred_logger); this->wellState(), deferred_logger);
} }
} }
OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::updateAndCommunicate failed: ", OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::updateAndCommunicate failed: ",
simulator_.gridView().comm()) simulator_.gridView().comm())
this->updateAndCommunicateGroupData(reportStepIdx, iterationIdx, param_.nupcol_group_rate_tolerance_, deferred_logger); this->updateAndCommunicateGroupData(reportStepIdx,
iterationIdx,
param_.nupcol_group_rate_tolerance_,
deferred_logger);
} }
template<typename TypeTag> template<typename TypeTag>