OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

cosmetics

Browse Source
This commit is contained in:
Arne Morten Kvarving 2025-01-22 16:16:42 +01:00
parent e288a61c59
commit 76c947b04c
8 changed files with 149 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
opm/simulators/aquifers/BlackoilAquiferModel_impl.hpp
View File

@ -139,8 +139,9 @@ BlackoilAquiferModel<TypeTag>::endTimeStep()
for (auto& aquifer : this->aquifers) { for (auto& aquifer : this->aquifers) {
aquifer->endTimeStep(); aquifer->endTimeStep();
NumAq* num = dynamic_cast<NumAq*>(aquifer.get()); NumAq* num = dynamic_cast<NumAq*>(aquifer.get());
if (num) if (num) {
this->simulator_.vanguard().grid().comm().barrier(); this->simulator_.vanguard().grid().comm().barrier();
}
} }
} }

43
opm/simulators/flow/GenericThresholdPressure_impl.hpp
View File

@ -73,10 +73,11 @@ thresholdPressure(int elem1Idx, int elem2Idx) const
int fault1Idx = lookUpCartesianData_(elem1Idx, cartElemFaultIdx_); int fault1Idx = lookUpCartesianData_(elem1Idx, cartElemFaultIdx_);
int fault2Idx = lookUpCartesianData_(elem2Idx, cartElemFaultIdx_); int fault2Idx = lookUpCartesianData_(elem2Idx, cartElemFaultIdx_);
if (fault1Idx != -1 && fault1Idx == fault2Idx) if (fault1Idx != -1 && fault1Idx == fault2Idx) {
// inside a fault there's no threshold pressure, even accross EQUIL // inside a fault there's no threshold pressure, even accross EQUIL
// regions. // regions.
return 0.0; return 0.0;
}
if (fault1Idx != fault2Idx) { if (fault1Idx != fault2Idx) {
// TODO: which value if a cell is part of multiple faults? we take // TODO: which value if a cell is part of multiple faults? we take
// the maximum here. // the maximum here.
@ -90,8 +91,9 @@ thresholdPressure(int elem1Idx, int elem2Idx) const
auto equilRegion1Idx = elemEquilRegion_[elem1Idx]; auto equilRegion1Idx = elemEquilRegion_[elem1Idx];
auto equilRegion2Idx = elemEquilRegion_[elem2Idx]; auto equilRegion2Idx = elemEquilRegion_[elem2Idx];
if (equilRegion1Idx == equilRegion2Idx) if (equilRegion1Idx == equilRegion2Idx) {
return 0.0; return 0.0;
}
return thpres_[equilRegion1Idx*numEquilRegions_ + equilRegion2Idx]; return thpres_[equilRegion1Idx*numEquilRegions_ + equilRegion2Idx];
} }
@ -103,8 +105,9 @@ finishInit()
const auto& simConfig = eclState_.getSimulationConfig(); const auto& simConfig = eclState_.getSimulationConfig();
enableThresholdPressure_ = simConfig.useThresholdPressure(); enableThresholdPressure_ = simConfig.useThresholdPressure();
if (!enableThresholdPressure_) if (!enableThresholdPressure_) {
return; return;
}
numEquilRegions_ = eclState_.getTableManager().getEqldims().getNumEquilRegions(); numEquilRegions_ = eclState_.getTableManager().getEqldims().getNumEquilRegions();
const decltype(numEquilRegions_) maxRegions = const decltype(numEquilRegions_) maxRegions =
@ -129,16 +132,18 @@ finishInit()
} }
// internalize the data specified using the EQLNUM keyword // internalize the data specified using the EQLNUM keyword
elemEquilRegion_ = lookUpData_.template assignFieldPropsIntOnLeaf<short unsigned int>(eclState_.fieldProps(), elemEquilRegion_ = lookUpData_.
"EQLNUM", true); template assignFieldPropsIntOnLeaf<short unsigned int>(eclState_.fieldProps(),
"EQLNUM", true);
/* /*
If this is a restart run the ThresholdPressure object will be active, If this is a restart run the ThresholdPressure object will be active,
and already properly initialized with numerical values from the restart. and already properly initialized with numerical values from the restart.
Done using GenericThresholdPressure::setFromRestart() in EclWriter::beginRestart(). Done using GenericThresholdPressure::setFromRestart() in EclWriter::beginRestart().
*/ */
if (simConfig.getThresholdPressure().restart()) if (simConfig.getThresholdPressure().restart()) {
return; return;
}
// allocate the array which specifies the threshold pressures // allocate the array which specifies the threshold pressures
thpres_.resize(numEquilRegions_*numEquilRegions_, 0.0); thpres_.resize(numEquilRegions_*numEquilRegions_, 0.0);
@ -156,10 +161,12 @@ applyExplicitThresholdPressures_()
// intersection in the grid // intersection in the grid
for (const auto& elem : elements(gridView_, Dune::Partitions::interior)) { for (const auto& elem : elements(gridView_, Dune::Partitions::interior)) {
for (const auto& intersection : intersections(gridView_, elem)) { for (const auto& intersection : intersections(gridView_, elem)) {
if (intersection.boundary()) if (intersection.boundary()) {
continue; // ignore boundary intersections for now (TODO?) continue; // ignore boundary intersections for now (TODO?)
else if (!intersection.neighbor()) //processor boundary but not domain boundary }
else if (!intersection.neighbor()) { // processor boundary but not domain boundary
continue; continue;
}
const auto& inside = intersection.inside(); const auto& inside = intersection.inside();
const auto& outside = intersection.outside(); const auto& outside = intersection.outside();
@ -189,8 +196,9 @@ applyExplicitThresholdPressures_()
} }
// apply threshold pressures across faults // apply threshold pressures across faults
if (thpres.ftSize() > 0) if (thpres.ftSize() > 0) {
configureThpresft_(); configureThpresft_();
}
} }
template<class Grid, class GridView, class ElementMapper, class Scalar> template<class Grid, class GridView, class ElementMapper, class Scalar>
@ -207,14 +215,16 @@ configureThpresft_()
int numCartesianElem = eclState_.getInputGrid().getCartesianSize(); int numCartesianElem = eclState_.getInputGrid().getCartesianSize();
thpresftValues_.resize(numFaults, -1.0); thpresftValues_.resize(numFaults, -1.0);
cartElemFaultIdx_.resize(numCartesianElem, -1); cartElemFaultIdx_.resize(numCartesianElem, -1);
for (std::size_t faultIdx = 0; faultIdx < faults.size(); faultIdx++) { for (std::size_t faultIdx = 0; faultIdx < faults.size(); ++faultIdx) {
auto& fault = faults.getFault(faultIdx); auto& fault = faults.getFault(faultIdx);
thpresftValues_[faultIdx] = thpres.getThresholdPressureFault(faultIdx); thpresftValues_[faultIdx] = thpres.getThresholdPressureFault(faultIdx);
for (const FaultFace& face : fault) for (const FaultFace& face : fault) {
// "face" is a misnomer because the object describes a set of cell // "face" is a misnomer because the object describes a set of cell
// indices, but we go with the conventions of the parser here... // indices, but we go with the conventions of the parser here...
for (std::size_t cartElemIdx : face) for (std::size_t cartElemIdx : face) {
cartElemFaultIdx_[cartElemIdx] = faultIdx; cartElemFaultIdx_[cartElemIdx] = faultIdx;
}
}
} }
} }
@ -223,8 +233,9 @@ std::vector<Scalar>
GenericThresholdPressure<Grid,GridView,ElementMapper,Scalar>:: GenericThresholdPressure<Grid,GridView,ElementMapper,Scalar>::
getRestartVector() const getRestartVector() const
{ {
if (!enableThresholdPressure_) if (!enableThresholdPressure_) {
return {}; return {};
}
return this->thpres_; return this->thpres_;
} }
@ -243,8 +254,9 @@ void
GenericThresholdPressure<Grid,GridView,ElementMapper,Scalar>:: GenericThresholdPressure<Grid,GridView,ElementMapper,Scalar>::
logPressures() logPressures()
{ {
if (!enableThresholdPressure_) if (!enableThresholdPressure_) {
return; return;
}
auto lineFormat = [this](unsigned i, unsigned j, double val) auto lineFormat = [this](unsigned i, unsigned j, double val)
{ {
@ -274,7 +286,8 @@ logPressures()
if (thpres.hasRegionBarrier(i, j)) { if (thpres.hasRegionBarrier(i, j)) {
if (thpres.hasThresholdPressure(i, j)) { if (thpres.hasThresholdPressure(i, j)) {
str += lineFormat(i, j, thpres.getThresholdPressure(j, i)); str += lineFormat(i, j, thpres.getThresholdPressure(j, i));
} else { }
else {
std::size_t idx = (j - 1) * numEquilRegions_ + (i - 1); std::size_t idx = (j - 1) * numEquilRegions_ + (i - 1);
str += lineFormat(i, j, this->thpresDefault_[idx]); str += lineFormat(i, j, this->thpresDefault_[idx]);
} }

2
opm/simulators/flow/TracerModel.hpp
View File

@ -892,7 +892,7 @@ protected:
TracerBatch(int phaseIdx = 0) : phaseIdx_(phaseIdx) {} TracerBatch(int phaseIdx = 0) : phaseIdx_(phaseIdx) {}
int numTracer() const {return idx_.size(); } int numTracer() const { return idx_.size(); }
void addTracer(const int idx, const TV & concentration) void addTracer(const int idx, const TV & concentration)
{ {

74
opm/simulators/linalg/gpubridge/Reorder.cpp
View File

@ -23,28 +23,28 @@
#include <opm/simulators/linalg/gpubridge/Reorder.hpp> #include <opm/simulators/linalg/gpubridge/Reorder.hpp>
#include <vector>
#include <cassert> #include <cassert>
#include <vector>
namespace Opm::Accelerator {
namespace Opm
{
namespace Accelerator
{
/* Check is operations on a node in the matrix can be started /* Check is operations on a node in the matrix can be started
* A node can only be started if all nodes that it depends on during sequential execution have already completed.*/ * A node can only be started if all nodes that it depends on during sequential
* execution have already completed.*/
bool canBeStarted(const int rowIndex, const int *rowPointers, const int *colIndices, const std::vector<bool>& doneRows) { bool canBeStarted(const int rowIndex,
const int* rowPointers,
const int* colIndices,
const std::vector<bool>& doneRows)
{
bool canStart = !doneRows[rowIndex]; bool canStart = !doneRows[rowIndex];
int i, thisDependency;
if (canStart) { if (canStart) {
for (i = rowPointers[rowIndex]; i < rowPointers[rowIndex + 1]; i++) { for (int i = rowPointers[rowIndex]; i < rowPointers[rowIndex + 1]; ++i) {
thisDependency = colIndices[i]; int thisDependency = colIndices[i];
// Only dependencies on rows that should execute before the current one are relevant // Only dependencies on rows that should execute before the current one are relevant
if (thisDependency >= rowIndex) if (thisDependency >= rowIndex) {
break; break;
}
// Check if dependency has been resolved // Check if dependency has been resolved
if (!doneRows[thisDependency]) { if (!doneRows[thisDependency]) {
return false; return false;
@ -55,14 +55,23 @@ bool canBeStarted(const int rowIndex, const int *rowPointers, const int *colIndi
} }
/* /*
* The level scheduling of a non-symmetric, blocked matrix requires access to a CSC encoding and a CSR encoding of the sparsity pattern of the input matrix. * The level scheduling of a non-symmetric, blocked matrix requires access to a CSC
* encoding and a CSR encoding of the sparsity pattern of the input matrix.
* This function is based on a standard level scheduling algorithm, like the one described in: * This function is based on a standard level scheduling algorithm, like the one described in:
* "Iterative methods for Sparse Linear Systems" by Yousef Saad in section 11.6.3 * "Iterative methods for Sparse Linear Systems" by Yousef Saad in section 11.6.3
*/ */
void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices, int *CSCColPointers, int Nb, int *numColors, int *toOrder, int* fromOrder, std::vector<int>& rowsPerColor) { void findLevelScheduling(int* CSRColIndices,
int activeRowIndex = 0, colorEnd, nextActiveRowIndex = 0; int* CSRRowPointers,
int thisRow; int* CSCRowIndices,
int* CSCColPointers,
int Nb,
int* numColors,
int* toOrder,
int* fromOrder,
std::vector<int>& rowsPerColor)
{
int activeRowIndex = 0, nextActiveRowIndex = 0;
std::vector<bool> doneRows(Nb, false); std::vector<bool> doneRows(Nb, false);
std::vector <int> rowsToStart; std::vector <int> rowsToStart;
@ -70,23 +79,27 @@ void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowInd
assert(rowsPerColor.empty()); assert(rowsPerColor.empty());
// find starting rows: rows that are independent from all rows that come before them. // find starting rows: rows that are independent from all rows that come before them.
for (thisRow = 0; thisRow < Nb; thisRow++) { int thisRow;
for (thisRow = 0; thisRow < Nb; ++thisRow) {
if (canBeStarted(thisRow, CSCColPointers, CSCRowIndices, doneRows)) { if (canBeStarted(thisRow, CSCColPointers, CSCRowIndices, doneRows)) {
fromOrder[nextActiveRowIndex] = thisRow; fromOrder[nextActiveRowIndex] = thisRow;
toOrder[thisRow] = nextActiveRowIndex; toOrder[thisRow] = nextActiveRowIndex;
nextActiveRowIndex++; ++nextActiveRowIndex;
} }
} }
// 'do' compute on all active rows // 'do' compute on all active rows
for (colorEnd = 0; colorEnd < nextActiveRowIndex; colorEnd++) { int colorEnd;
for (colorEnd = 0; colorEnd < nextActiveRowIndex; ++colorEnd) {
doneRows[fromOrder[colorEnd]] = true; doneRows[fromOrder[colorEnd]] = true;
} }
rowsPerColor.emplace_back(nextActiveRowIndex - activeRowIndex); rowsPerColor.emplace_back(nextActiveRowIndex - activeRowIndex);
while (colorEnd < Nb) { while (colorEnd < Nb) {
// Go over all rows active from the last color, and check which of their neighbours can be activated this color // Go over all rows active from the last color, and check which of
for (; activeRowIndex < colorEnd; activeRowIndex++) { // their neighbours can be activated this color
for (; activeRowIndex < colorEnd; ++activeRowIndex) {
thisRow = fromOrder[activeRowIndex]; thisRow = fromOrder[activeRowIndex];
for (int i = CSCColPointers[thisRow]; i < CSCColPointers[thisRow + 1]; i++) { for (int i = CSCColPointers[thisRow]; i < CSCColPointers[thisRow + 1]; i++) {
@ -104,7 +117,7 @@ void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowInd
doneRows[thisRow] = true; doneRows[thisRow] = true;
fromOrder[nextActiveRowIndex] = thisRow; fromOrder[nextActiveRowIndex] = thisRow;
toOrder[thisRow] = nextActiveRowIndex; toOrder[thisRow] = nextActiveRowIndex;
nextActiveRowIndex++; ++nextActiveRowIndex;
} }
} }
rowsToStart.clear(); rowsToStart.clear();
@ -115,10 +128,13 @@ void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowInd
*numColors = rowsPerColor.size(); *numColors = rowsPerColor.size();
} }
// based on the scipy package from python, scipy/sparse/sparsetools/csr.h on github // based on the scipy package from python, scipy/sparse/sparsetools/csr.h on github
void csrPatternToCsc(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices, int *CSCColPointers, int Nb) { void csrPatternToCsc(int* CSRColIndices,
int* CSRRowPointers,
int* CSCRowIndices,
int* CSCColPointers,
int Nb)
{
int nnz = CSRRowPointers[Nb]; int nnz = CSRRowPointers[Nb];
// compute number of nnzs per column // compute number of nnzs per column
@ -141,7 +157,7 @@ void csrPatternToCsc(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices
int col = CSRColIndices[j]; int col = CSRColIndices[j];
int dest = CSCColPointers[col]; int dest = CSCColPointers[col];
CSCRowIndices[dest] = row; CSCRowIndices[dest] = row;
CSCColPointers[col]++; ++CSCColPointers[col];
} }
} }
@ -152,6 +168,4 @@ void csrPatternToCsc(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices
} }
} }
} // namespace Opm::Accelerator
} // namespace Accelerator
} // namespace Opm

31
opm/simulators/linalg/gpubridge/Reorder.hpp
View File

@ -22,10 +22,7 @@
#include <vector> #include <vector>
namespace Opm namespace Opm::Accelerator {
{
namespace Accelerator
{
/// Determine whether all rows that a certain row depends on are done already /// Determine whether all rows that a certain row depends on are done already
/// \param[in] rowIndex index of the row that needs to be checked for /// \param[in] rowIndex index of the row that needs to be checked for
@ -33,7 +30,10 @@ namespace Accelerator
/// \param[in] colIndices column indices of the matrix that the row is in /// \param[in] colIndices column indices of the matrix that the row is in
/// \param[in] doneRows array that for each row lists whether it is done or not /// \param[in] doneRows array that for each row lists whether it is done or not
/// \return true iff all dependencies are done and if the result itself was not done yet /// \return true iff all dependencies are done and if the result itself was not done yet
bool canBeStarted(const int rowIndex, const int *rowPointers, const int *colIndices, const std::vector<bool>& doneRows); bool canBeStarted(const int rowIndex,
const int* rowPointers,
const int* colIndices,
const std::vector<bool>& doneRows);
/// Find a level scheduling reordering for an input matrix /// Find a level scheduling reordering for an input matrix
/// The toOrder and fromOrder arrays must be allocated already /// The toOrder and fromOrder arrays must be allocated already
@ -46,7 +46,15 @@ bool canBeStarted(const int rowIndex, const int *rowPointers, const int *colIn
/// \param[out] toOrder the reorder pattern that was found, which lists for each index in the original order, to which index in the new order it should be moved /// \param[out] toOrder the reorder pattern that was found, which lists for each index in the original order, to which index in the new order it should be moved
/// \param[out] fromOrder the reorder pattern that was found, which lists for each index in the new order, from which index in the original order it was moved /// \param[out] fromOrder the reorder pattern that was found, which lists for each index in the new order, from which index in the original order it was moved
/// \param[out] rowsPerColor for each color, an array of all rowIndices in that color, this function uses emplace_back() to fill /// \param[out] rowsPerColor for each color, an array of all rowIndices in that color, this function uses emplace_back() to fill
void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices, int *CSCColPointers, int Nb, int *numColors, int *toOrder, int* fromOrder, std::vector<int>& rowsPerColor); void findLevelScheduling(int* CSRColIndices,
int* CSRRowPointers,
int* CSCRowIndices,
int* CSCColPointers,
int Nb,
int* numColors,
int* toOrder,
int* fromOrder,
std::vector<int>& rowsPerColor);
/// Convert a sparsity pattern stored in the CSR format to the CSC format /// Convert a sparsity pattern stored in the CSR format to the CSC format
/// CSCRowIndices and CSCColPointers arrays must be allocated already /// CSCRowIndices and CSCColPointers arrays must be allocated already
@ -56,9 +64,12 @@ void findLevelScheduling(int *CSRColIndices, int *CSRRowPointers, int *CSCRowInd
/// \param[inout] CSCRowIndices row indices of the result CSC representation of the pattern /// \param[inout] CSCRowIndices row indices of the result CSC representation of the pattern
/// \param[inout] CSCColPointers column pointers of the result CSC representation of the pattern /// \param[inout] CSCColPointers column pointers of the result CSC representation of the pattern
/// \param[in] Nb number of blockrows in the matrix /// \param[in] Nb number of blockrows in the matrix
void csrPatternToCsc(int *CSRColIndices, int *CSRRowPointers, int *CSCRowIndices, int *CSCColPointers, int Nb); void csrPatternToCsc(int* CSRColIndices,
int* CSRRowPointers,
int* CSCRowIndices,
int* CSCColPointers,
int Nb);
} // namespace Accelerator } // namespace Opm::Accelerator
} // namespace Opm
#endif #endif

3
opm/simulators/linalg/gpubridge/opencl/openclBILU0.cpp
View File

@ -198,8 +198,9 @@ create_preconditioner(BlockedMatrix<Scalar>* mat, BlockedMatrix<Scalar>* jacMat)
bool use_multithreading = true; bool use_multithreading = true;
#if HAVE_OPENMP #if HAVE_OPENMP
if (omp_get_max_threads() == 1) if (omp_get_max_threads() == 1) {
use_multithreading = false; use_multithreading = false;
}
#endif #endif
if (jacMat && use_multithreading) { if (jacMat && use_multithreading) {

69
opm/simulators/linalg/gpuistl/GpuOwnerOverlapCopy.hpp
View File

@ -24,8 +24,8 @@
#include <opm/simulators/linalg/gpuistl/GpuVector.hpp> #include <opm/simulators/linalg/gpuistl/GpuVector.hpp>
#include <vector> #include <vector>
namespace Opm::gpuistl namespace Opm::gpuistl {
{
/** /**
* @brief GPUSender is a wrapper class for classes which will implement copOwnerToAll * @brief GPUSender is a wrapper class for classes which will implement copOwnerToAll
* This is implemented with the intention of creating communicators with generic GPUSender * This is implemented with the intention of creating communicators with generic GPUSender
@ -119,10 +119,11 @@ public:
explicit GPUObliviousMPISender(const OwnerOverlapCopyCommunicationType& cpuOwnerOverlapCopy) explicit GPUObliviousMPISender(const OwnerOverlapCopyCommunicationType& cpuOwnerOverlapCopy)
: GPUSender<field_type, OwnerOverlapCopyCommunicationType>(cpuOwnerOverlapCopy) : GPUSender<field_type, OwnerOverlapCopyCommunicationType>(cpuOwnerOverlapCopy)
{ {
} }
void copyOwnerToAll(const X& source, X& dest) const override { void copyOwnerToAll(const X& source, X& dest) const override
{
// TODO: [perf] Can we reduce copying from the GPU here? // TODO: [perf] Can we reduce copying from the GPU here?
// TODO: [perf] Maybe create a global buffer instead? // TODO: [perf] Maybe create a global buffer instead?
auto sourceAsDuneVector = source.template asDuneBlockVector<block_size>(); auto sourceAsDuneVector = source.template asDuneBlockVector<block_size>();
@ -179,7 +180,6 @@ public:
void copyOwnerToAll(const X& source, X& dest) const override void copyOwnerToAll(const X& source, X& dest) const override
{ {
OPM_ERROR_IF(&source != &dest, "The provided GpuVectors' address did not match"); // In this context, source == dest!!! OPM_ERROR_IF(&source != &dest, "The provided GpuVectors' address did not match"); // In this context, source == dest!!!
std::call_once(this->m_initializedIndices, [&]() { initIndexSet(); }); std::call_once(this->m_initializedIndices, [&]() { initIndexSet(); });
@ -198,9 +198,9 @@ public:
{ {
size_t i = 0; size_t i = 0;
for(const_iterator info = m_messageInformation.begin(); info != end; ++info, ++i) { for (const_iterator info = m_messageInformation.begin(); info != end; ++info, ++i) {
processMap[i]=info->first; processMap[i]=info->first;
if(info->second.second.m_size) { if (info->second.second.m_size) {
MPI_Irecv(m_GPURecvBuf->data()+info->second.second.m_start, MPI_Irecv(m_GPURecvBuf->data()+info->second.second.m_start,
detail::to_int(info->second.second.m_size), detail::to_int(info->second.second.m_size),
MPI_BYTE, MPI_BYTE,
@ -209,16 +209,17 @@ public:
this->m_cpuOwnerOverlapCopy.communicator(), this->m_cpuOwnerOverlapCopy.communicator(),
&recvRequests[i]); &recvRequests[i]);
numberOfRealRecvRequests += 1; numberOfRealRecvRequests += 1;
} else { }
recvRequests[i]=MPI_REQUEST_NULL; else {
recvRequests[i] = MPI_REQUEST_NULL;
} }
} }
} }
{ {
size_t i = 0; size_t i = 0;
for(const_iterator info = m_messageInformation.begin(); info != end; ++info, ++i) { for (const_iterator info = m_messageInformation.begin(); info != end; ++info, ++i) {
if(info->second.first.m_size) { if (info->second.first.m_size) {
MPI_Issend(m_GPUSendBuf->data()+info->second.first.m_start, MPI_Issend(m_GPUSendBuf->data()+info->second.first.m_start,
detail::to_int(info->second.first.m_size), detail::to_int(info->second.first.m_size),
MPI_BYTE, MPI_BYTE,
@ -227,24 +228,28 @@ public:
this->m_cpuOwnerOverlapCopy.communicator(), this->m_cpuOwnerOverlapCopy.communicator(),
&sendRequests[i]); &sendRequests[i]);
} else { } else {
sendRequests[i]=MPI_REQUEST_NULL; sendRequests[i] = MPI_REQUEST_NULL;
} }
} }
} }
int finished = MPI_UNDEFINED; int finished = MPI_UNDEFINED;
MPI_Status status; MPI_Status status;
for(size_t i = 0; i < numberOfRealRecvRequests; i++) { for (size_t i = 0; i < numberOfRealRecvRequests; i++) {
status.MPI_ERROR=MPI_SUCCESS; status.MPI_ERROR=MPI_SUCCESS;
MPI_Waitany(m_messageInformation.size(), recvRequests.data(), &finished, &status); MPI_Waitany(m_messageInformation.size(), recvRequests.data(), &finished, &status);
if(status.MPI_ERROR!=MPI_SUCCESS) { if (status.MPI_ERROR!=MPI_SUCCESS) {
OPM_THROW(std::runtime_error, fmt::format("MPI_Error occurred while rank {} received a message from rank {}", rank, processMap[finished])); OPM_THROW(std::runtime_error,
fmt::format("MPI_Error occurred while rank {} received a message from rank {}",
rank, processMap[finished]));
} }
} }
MPI_Status recvStatus; MPI_Status recvStatus;
for(size_t i = 0; i < m_messageInformation.size(); i++) { for (size_t i = 0; i < m_messageInformation.size(); i++) {
if(MPI_SUCCESS!=MPI_Wait(&sendRequests[i], &recvStatus)) { if (MPI_SUCCESS != MPI_Wait(&sendRequests[i], &recvStatus)) {
OPM_THROW(std::runtime_error, fmt::format("MPI_Error occurred while rank {} sent a message from rank {}", rank, processMap[finished])); OPM_THROW(std::runtime_error,
fmt::format("MPI_Error occurred while rank {} sent a message from rank {}",
rank, processMap[finished]));
} }
} }
// ...End of MPI stuff // ...End of MPI stuff
@ -279,20 +284,21 @@ private:
std::vector<int> commpairIndicesCopyOnCPU; std::vector<int> commpairIndicesCopyOnCPU;
std::vector<int> commpairIndicesOwnerCPU; std::vector<int> commpairIndicesOwnerCPU;
for(auto process : ri) { for (auto process : ri) {
m_im[process.first] = std::pair(std::vector<int>(), std::vector<int>()); m_im[process.first] = std::pair(std::vector<int>(), std::vector<int>());
for(int send = 0; send < 2; ++send) { for (int send = 0; send < 2; ++send) {
auto remoteEnd = send ? process.second.first->end() auto remoteEnd = send ? process.second.first->end()
: process.second.second->end(); : process.second.second->end();
auto remote = send ? process.second.first->begin() auto remote = send ? process.second.first->begin()
: process.second.second->begin(); : process.second.second->begin();
while(remote != remoteEnd) { while (remote != remoteEnd) {
if (send ? (remote->localIndexPair().local().attribute() == 1) if (send ? (remote->localIndexPair().local().attribute() == 1)
: (remote->attribute() == 1)) { : (remote->attribute() == 1)) {
if (send) { if (send) {
m_im[process.first].first.push_back(remote->localIndexPair().local().local()); m_im[process.first].first.push_back(remote->localIndexPair().local().local());
} else { }
else {
m_im[process.first].second.push_back(remote->localIndexPair().local().local()); m_im[process.first].second.push_back(remote->localIndexPair().local().local());
} }
} }
@ -317,13 +323,13 @@ private:
recvBufIdx * block_size, recvBufIdx * block_size,
noRecv * block_size * sizeof(field_type))))); noRecv * block_size * sizeof(field_type)))));
for(int x = 0; x < noSend; x++) { for (int x = 0; x < noSend; x++) {
for(int bs = 0; bs < block_size; bs++) { for (int bs = 0; bs < block_size; bs++) {
commpairIndicesOwnerCPU.push_back(it->second.first[x] * block_size + bs); commpairIndicesOwnerCPU.push_back(it->second.first[x] * block_size + bs);
} }
} }
for(int x = 0; x < noRecv; x++) { for (int x = 0; x < noRecv; x++) {
for(int bs = 0; bs < block_size; bs++) { for (int bs = 0; bs < block_size; bs++) {
commpairIndicesCopyOnCPU.push_back(it->second.second[x] * block_size + bs); commpairIndicesCopyOnCPU.push_back(it->second.second[x] * block_size + bs);
} }
} }
@ -385,9 +391,12 @@ class GpuOwnerOverlapCopy
public: public:
using X = GpuVector<field_type>; using X = GpuVector<field_type>;
explicit GpuOwnerOverlapCopy(std::shared_ptr<GPUSender<field_type, OwnerOverlapCopyCommunicationType>> sender) : m_sender(sender){} explicit GpuOwnerOverlapCopy(std::shared_ptr<GPUSender<field_type, OwnerOverlapCopyCommunicationType>> sender)
: m_sender(sender)
{}
void copyOwnerToAll(const X& source, X& dest) const { void copyOwnerToAll(const X& source, X& dest) const
{
m_sender->copyOwnerToAll(source, dest); m_sender->copyOwnerToAll(source, dest);
} }
@ -409,5 +418,7 @@ public:
private: private:
std::shared_ptr<GPUSender<field_type, OwnerOverlapCopyCommunicationType>> m_sender; std::shared_ptr<GPUSender<field_type, OwnerOverlapCopyCommunicationType>> m_sender;
}; };
} // namespace Opm::gpuistl } // namespace Opm::gpuistl
#endif #endif

14
opm/simulators/wells/BlackoilWellModel_impl.hpp
View File

@ -2011,7 +2011,10 @@ namespace Opm {
const int iterationIdx, const int iterationIdx,
DeferredLogger& deferred_logger) DeferredLogger& deferred_logger)
{ {
this->updateAndCommunicateGroupData(reportStepIdx, iterationIdx, param_.nupcol_group_rate_tolerance_, deferred_logger); this->updateAndCommunicateGroupData(reportStepIdx,
iterationIdx,
param_.nupcol_group_rate_tolerance_,
deferred_logger);
// updateWellStateWithTarget might throw for multisegment wells hence we // updateWellStateWithTarget might throw for multisegment wells hence we
// have a parallel try catch here to thrown on all processes. // have a parallel try catch here to thrown on all processes.
@ -2020,14 +2023,19 @@ namespace Opm {
for (const auto& well : well_container_) { for (const auto& well : well_container_) {
// We only want to update wells under group-control here // We only want to update wells under group-control here
auto& ws = this->wellState().well(well->indexOfWell()); auto& ws = this->wellState().well(well->indexOfWell());
if (ws.production_cmode == Well::ProducerCMode::GRUP || ws.injection_cmode == Well::InjectorCMode::GRUP) { if (ws.production_cmode == Well::ProducerCMode::GRUP ||
ws.injection_cmode == Well::InjectorCMode::GRUP)
{
well->updateWellStateWithTarget(simulator_, this->groupState(), well->updateWellStateWithTarget(simulator_, this->groupState(),
this->wellState(), deferred_logger); this->wellState(), deferred_logger);
} }
} }
OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::updateAndCommunicate failed: ", OPM_END_PARALLEL_TRY_CATCH("BlackoilWellModel::updateAndCommunicate failed: ",
simulator_.gridView().comm()) simulator_.gridView().comm())
this->updateAndCommunicateGroupData(reportStepIdx, iterationIdx, param_.nupcol_group_rate_tolerance_, deferred_logger); this->updateAndCommunicateGroupData(reportStepIdx,
iterationIdx,
param_.nupcol_group_rate_tolerance_,
deferred_logger);
} }
template<typename TypeTag> template<typename TypeTag>