OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

hopefully fixed compilation and linking problems with WITHGPU

Browse Source
This commit is contained in:
hnil 2023-08-09 14:54:10 +02:00
parent 63b9b01671
commit 07fb18422d
7 changed files with 91 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists_files.cmake
View File

@ -62,6 +62,7 @@ list (APPEND MAIN_SOURCE_FILES
opm/simulators/linalg/FlexibleSolver5.cpp
opm/simulators/linalg/FlexibleSolver6.cpp
opm/simulators/linalg/ISTLSolverEbos.cpp
opm/simulators/linalg/ISTLSolverEbosWithGPU.cpp
opm/simulators/linalg/MILU.cpp
opm/simulators/linalg/ParallelIstlInformation.cpp
opm/simulators/linalg/ParallelOverlappingILU0.cpp
@ -143,7 +144,7 @@ list (APPEND MAIN_SOURCE_FILES
)
if (DAMARIS_FOUND AND MPI_FOUND)
list (APPEND MAIN_SOURCE_FILES opm/simulators/utils/DamarisOutputModule.cpp)
list (APPEND MAIN_SOURCE_FILES opm/simulators/utils/DamarisOutputModule.cpp)
list (APPEND MAIN_SOURCE_FILES opm/simulators/utils/DamarisKeywords.cpp)
list (APPEND MAIN_SOURCE_FILES opm/simulators/utils/initDamarisXmlFile.cpp)
endif()
@ -175,7 +176,7 @@ if(CUDA_FOUND)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/safe_conversion.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cublas_wrapper.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cusparse_wrapper.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cusparse_constants.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cusparse_constants.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/vector_operations.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/has_function.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/preconditioner_should_call_post_pre.hpp)
@ -462,6 +463,7 @@ list (APPEND PUBLIC_HEADER_FILES
opm/simulators/linalg/FlowLinearSolverParameters.hpp
opm/simulators/linalg/GraphColoring.hpp
opm/simulators/linalg/ISTLSolverEbos.hpp
opm/simulators/linalg/ISTLSolverEbosWithGPU.hpp
opm/simulators/linalg/MatrixMarketSpecializations.hpp
opm/simulators/linalg/OwningBlockPreconditioner.hpp
opm/simulators/linalg/OwningTwoLevelPreconditioner.hpp

2
opm/simulators/flow/BlackoilModelEbosNldd.hpp
View File

@ -34,7 +34,7 @@
#include <opm/simulators/flow/SubDomain.hpp>
#include <opm/simulators/linalg/extractMatrix.hpp>
#include <opm/simulators/linalg/ISTLSolverEbos.hpp>
#include <opm/simulators/linalg/ISTLSolverEbosWithGPU.hpp>
#include <opm/simulators/timestepping/ConvergenceReport.hpp>
#include <opm/simulators/timestepping/SimulatorReport.hpp>

12
opm/simulators/linalg/FlowLinearSolverParameters.hpp
View File

@ -31,10 +31,13 @@
namespace Opm {
template <class TypeTag>
class ISTLSolverEbosWithGPU;
template <class TypeTag>
class ISTLSolverEbos;
}
namespace Opm::Properties {
namespace TTag {
@ -50,7 +53,7 @@ template<class TypeTag, class MyTypeTag>
struct RelaxedLinearSolverReduction {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct LinearSolverMaxIter {
using type = UndefinedProperty;
@ -135,7 +138,7 @@ template<class TypeTag>
struct RelaxedLinearSolverReduction<TypeTag, TTag::FlowIstlSolverParams> {
using type = GetPropType<TypeTag, Scalar>;
static constexpr type value = 1e-2;
};
};
template<class TypeTag>
struct LinearSolverMaxIter<TypeTag, TTag::FlowIstlSolverParams> {
static constexpr int value = 200;
@ -217,9 +220,12 @@ struct OpenclIluParallel<TypeTag, TTag::FlowIstlSolverParams> {
// Set the backend to be used.
template<class TypeTag>
struct LinearSolverBackend<TypeTag, TTag::FlowIstlSolverParams> {
#if COMPLE_BDA_BRIDGE
using type = ISTLSolverEbosWithGPU<TypeTag>;
#else
using type = ISTLSolverEbos<TypeTag>;
#endif
};
} // namespace Opm::Properties
namespace Opm

12
opm/simulators/linalg/ISTLSolverEbos.cpp
View File

@ -217,12 +217,12 @@ using CommunicationType = Dune::CollectiveCommunication<int>;
template void makeOverlapRowsInvalid<BM<Dim>>(BM<Dim>&, const std::vector<int>&); \
template struct FlexibleSolverInfo<BM<Dim>,BV<Dim>,CommunicationType>;
INSTANCE(1)
INSTANCE(2)
INSTANCE(3)
INSTANCE(4)
INSTANCE(5)
INSTANCE(6)
INSTANCE_FLEX(1)
INSTANCE_FLEX(2)
INSTANCE_FLEX(3)
INSTANCE_FLEX(4)
INSTANCE_FLEX(5)
INSTANCE_FLEX(6)
}
}

8
opm/simulators/linalg/ISTLSolverEbos.hpp
View File

@ -201,7 +201,7 @@ std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
initialize();
}
void initialize()
void initialize(bool have_gpu = false)
{
OPM_TIMEBLOCK(IstlSolverEbos);
prm_ = setupPropertyTree(parameters_,
@ -238,6 +238,12 @@ std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
prm_.write_json(os, true);
OpmLog::note(os.str());
}
if(have_gpu){
if (EWOMS_GET_PARAM(TypeTag, std::string, AcceleratorMode) != "none") {
OPM_THROW(std::logic_error,"Cannot use accelerated solver since CUDA, OpenCL and amgcl were not found by cmake");
}
}
}
// nothing to clean here

27
opm/simulators/linalg/ISTLSolverEbosWithGPU.cpp
View File

@ -21,7 +21,7 @@
#include <config.h>
#include <opm/common/TimingMacros.hpp>
#include <opm/simulators/linalg/ISTLSolverEbosGPU.hpp>
#include <opm/simulators/linalg/ISTLSolverEbosWithGPU.hpp>
#include <dune/istl/schwarz.hh>
@ -211,10 +211,15 @@ copyMatToBlockJac(const Matrix& mat, Matrix& blockJac)
}
#endif // COMPILE_BDA_BRIDGE
template<int Dim>
using BM = Dune::BCRSMatrix<MatrixBlock<double,Dim,Dim>>;
template<int Dim>
using BV = Dune::BlockVector<Dune::FieldVector<double,Dim>>;
#if COMPILE_BDA_BRIDGE
#define INSTANCE_GRID(Dim, Grid) \
#define INSTANCE_GRID(Dim, Grid) \
template void BdaSolverInfo<BM<Dim>,BV<Dim>>:: \
prepare(const Grid&, \
const Dune::CartesianIndexMapper<Grid>&, \
@ -232,19 +237,19 @@ using PolyHedralGrid3D = Dune::PolyhedralGrid<3, 3>;
template struct BdaSolverInfo<BM<Dim>,BV<Dim>>; \
INSTANCE_GRID(Dim,Dune::CpGrid) \
INSTANCE_GRID(Dim,ALUGrid3CN) \
INSTANCE_GRID(Dim,PolyHedralGrid3D) \
INSTANCE_FLEX(Dim)
INSTANCE_GRID(Dim,PolyHedralGrid3D)
#else
#define INSTANCE(Dim) \
template struct BdaSolverInfo<BM<Dim>,BV<Dim>>; \
INSTANCE_GRID(Dim,Dune::CpGrid) \
INSTANCE_GRID(Dim,PolyHedralGrid3D) \
INSTANCE_FLEX(Dim)
INSTANCE_GRID(Dim,PolyHedralGrid3D)
#endif
INSTANCE(1)
INSTANCE(2)
INSTANCE(3)
INSTANCE(4)
INSTANCE(5)
INSTANCE(6)
#endif
#else
#define INSTANCE(Dim) \
INSTANCE_FLEX(Dim)
#endif // COMPILE_BDA_BRIDGE
}
}

115
opm/simulators/linalg/ISTLSolverEbosWithGPU.hpp
View File

@ -19,20 +19,16 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPM_ISTLSOLVER_EBOS_GPU_HEADER_INCLUDED
#define OPM_ISTLSOLVER_EBOS_GPU_HEADER_INCLUDED
#ifndef OPM_ISTLSOLVER_EBOS_WITH_GPU_HEADER_INCLUDED
#define OPM_ISTLSOLVER_EBOS_WITH_GPU_HEADER_INCLUDED
#include <opm/simulators/linalg/ISTLSolverEbos.hpp>
#if COMPILE_BDA_BRIDGE
#include <opm/simulators/linalg/ISTLSolverEbosGPU.hpp>
namespace Opm::Properties {
namespace Opm
{
template<class Matrix, class Vector, int block_size> class BdaBridge;
class WellContributions;
namespace detail
{
template<class Matrix, class Vector>
struct BdaSolverInfo
{
@ -86,20 +82,16 @@ private:
std::vector<std::set<int>> wellConnectionsGraph_;
};
/// Create sparsity pattern for block-Jacobi matrix based on partitioning of grid.
/// Do not initialize the values, that is done in copyMatToBlockJac()
}
/// This class solves the fully implicit black-oil system by
/// solving the reduced system (after eliminating well variables)
/// as a block-structured matrix (one block for all cell variables) for a fixed
/// number of cell variables np .
template <class TypeTag>
class ISTLSolverEbosWithGPU : public ISTLSolverEbos
class ISTLSolverEbosWithGPU : public ISTLSolverEbos<TypeTag>
{
protected:
using ParentType = ISTLSolverEbos<TypeTag>;
using GridView = GetPropType<TypeTag, Properties::GridView>;
using Scalar = GetPropType<TypeTag, Properties::Scalar>;
using SparseMatrixAdapter = GetPropType<TypeTag, Properties::SparseMatrixAdapter>;
@ -130,33 +122,28 @@ private:
/// \param[in] simulator The opm-models simulator object
/// \param[in] parameters Explicit parameters for solver setup, do not
/// read them from command line parameters.
ISTLSolverEbosGPU(const Simulator& simulator, const FlowLinearSolverParameters& parameters)
: ISTLSolverEbos(simulator),
ISTLSolverEbosWithGPU(const Simulator& simulator, const FlowLinearSolverParameters& parameters)
: ParentType(simulator)
{
this->initialize();
bool have_gpu = true;
this->initialize(have_gpu);
}
/// Construct a system solver.
/// \param[in] simulator The opm-models simulator object
explicit ISTLSolverEbos(const Simulator& simulator)
: simulator_(simulator),
iterations_( 0 ),
calls_( 0 ),
converged_(false),
matrix_(nullptr)
explicit ISTLSolverEbosWithGPU(const Simulator& simulator)
: ParentType(simulator)
{
parameters_.template init<TypeTag>(simulator_.vanguard().eclState().getSimulationConfig().useCPR());
initialize();
}
void initialize()
{
OPM_TIMEBLOCK(IstlSolverEbos);
IstlSolverEbos::initialize()
#if COMPILE_BDA_BRIDGE
OPM_TIMEBLOCK(initialize);
ParentType::initialize(false);
const bool on_io_rank = (this->simulator_.gridView().comm().rank() == 0);
{
std::string accelerator_mode = EWOMS_GET_PARAM(TypeTag, std::string, AcceleratorMode);
if ((simulator_.vanguard().grid().comm().size() > 1) && (accelerator_mode != "none")) {
if ((this->simulator_.vanguard().grid().comm().size() > 1) && (accelerator_mode != "none")) {
if (on_io_rank) {
OpmLog::warning("Cannot use GPU with MPI, GPU are disabled");
}
@ -167,9 +154,9 @@ private:
const int maxit = EWOMS_GET_PARAM(TypeTag, int, LinearSolverMaxIter);
const double tolerance = EWOMS_GET_PARAM(TypeTag, double, LinearSolverReduction);
const bool opencl_ilu_parallel = EWOMS_GET_PARAM(TypeTag, bool, OpenclIluParallel);
const int linear_solver_verbosity = parameters_.linear_solver_verbosity_;
const int linear_solver_verbosity = this->parameters_.linear_solver_verbosity_;
std::string linsolver = EWOMS_GET_PARAM(TypeTag, std::string, LinearSolver);
bdaBridge = std::make_unique<detail::BdaSolverInfo<Matrix,Vector>>(accelerator_mode,
bdaBridge_ = std::make_unique<detail::BdaSolverInfo<Matrix,Vector>>(accelerator_mode,
linear_solver_verbosity,
maxit,
tolerance,
@ -178,18 +165,13 @@ private:
opencl_ilu_parallel,
linsolver);
}
#else
if (EWOMS_GET_PARAM(TypeTag, std::string, AcceleratorMode) != "none") {
OPM_THROW(std::logic_error,"Cannot use accelerated solver since CUDA, OpenCL and amgcl were not found by cmake");
}
#endif
}
void prepare(const Matrix& M, Vector& b)
{
OPM_TIMEBLOCK(istlSolverEbosPrepare);
IstlSolverEbos::prepare(M,b);
const bool firstcall = (matrix_ == nullptr);
OPM_TIMEBLOCK(prepare);
ParentType::prepare(M,b);
const bool firstcall = (this->matrix_ == nullptr);
// update matrix entries for solvers.
if (firstcall) {
// ebos will not change the matrix object. Hence simply store a pointer
@ -197,12 +179,12 @@ private:
// Outch! We need to be able to scale the linear system! Hence const_cast
// setup sparsity pattern for jacobi matrix for preconditioner (only used for openclSolver)
#if HAVE_OPENCL
bdaBridge->numJacobiBlocks_ = EWOMS_GET_PARAM(TypeTag, int, NumJacobiBlocks);
bdaBridge->prepare(simulator_.vanguard().grid(),
simulator_.vanguard().cartesianIndexMapper(),
simulator_.vanguard().schedule().getWellsatEnd(),
simulator_.vanguard().cellPartition(),
getMatrix().nonzeroes(), useWellConn_);
bdaBridge_->numJacobiBlocks_ = EWOMS_GET_PARAM(TypeTag, int, NumJacobiBlocks);
bdaBridge_->prepare(this->simulator_.vanguard().grid(),
this->simulator_.vanguard().cartesianIndexMapper(),
this->simulator_.vanguard().schedule().getWellsatEnd(),
this->simulator_.vanguard().cellPartition(),
this->getMatrix().nonzeroes(), this->useWellConn_);
#endif
}
}
@ -215,7 +197,7 @@ private:
void getResidual(Vector& b) const
{
b = *rhs_;
b = *(this->rhs_);
}
void setMatrix(const SparseMatrixAdapter& /* M */)
@ -225,16 +207,16 @@ private:
bool solve(Vector& x)
{
OPM_TIMEBLOCK(istlSolverEbosSolve);
calls_ += 1;
OPM_TIMEBLOCK(solve);
this->calls_ += 1;
// Write linear system if asked for.
const int verbosity = prm_.get<int>("verbosity", 0);
const int verbosity = this->prm_.template get<int>("verbosity", 0);
const bool write_matrix = verbosity > 10;
if (write_matrix) {
Helper::writeSystem(simulator_, //simulator is only used to get names
getMatrix(),
*rhs_,
comm_.get());
Helper::writeSystem(this->simulator_, //simulator is only used to get names
this->getMatrix(),
*(this->rhs_),
this->comm_.get());
}
// Solve system.
@ -245,36 +227,35 @@ private:
{
this->simulator_.problem().wellModel().getWellContributions(w);
};
if (!bdaBridge->apply(*rhs_, useWellConn_, getContribs,
simulator_.gridView().comm().rank(),
if (!bdaBridge_->apply(*(this->rhs_), this->useWellConn_, getContribs,
this->simulator_.gridView().comm().rank(),
const_cast<Matrix&>(this->getMatrix()),
x, result))
{
OPM_TIMEBLOCK(flexibleSolverApply);
if(bdaBridge->gpuActive()){
if(bdaBridge_->gpuActive()){
// bda solve fails use istl solver setup need to be done since it is not setup in prepeare
ISTLSolverEbos::prepareFlexibleSolver();
ParentType::prepareFlexibleSolver();
}
assert(flexibleSolver_.solver_);
flexibleSolver_.solver_->apply(x, *rhs_, result);
assert(this->flexibleSolver_.solver_);
this->flexibleSolver_.solver_->apply(x, *(this->rhs_), result);
}
// Check convergence, iterations etc.
checkConvergence(result);
this->checkConvergence(result);
return converged_;
return this->converged_;
}
protected:
void prepareFlexibleSolver()
{
if(bdaBridge->gpuActive()){
ISTLSolverEbos::prepareFlexibleSolver();
if(bdaBridge_->gpuActive()){
ParentType::prepareFlexibleSolver();
}
}
std::unique_ptr<detail::BdaSolverInfo<Matrix, Vector>> bdaBridge;
std::unique_ptr<detail::BdaSolverInfo<Matrix, Vector>> bdaBridge_;
}; // end ISTLSolver
} // namespace Opm
#endif //COMPILE_BDA
#endif
#endif