OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #538 from blattms/parallel-solver-support

Browse Source 
Added support for parallel dune-istl solvers
This commit is contained in:
Bård Skaflestad
2014-04-08 00:17:35 +02:00
parent 28abf88520 99d7f61822
commit 333e110d4f
9 changed files with 513 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
opm/core/linalg/LinearSolverFactory.cpp
View File

@@ -97,9 +97,10 @@ namespace Opm
const int* ja,
const double* sa,
const double* rhs,
double* solution) const
double* solution,
const boost::any& add) const
{
return solver_->solve(size, nonzeros, ia, ja, sa, rhs, solution);
return solver_->solve(size, nonzeros, ia, ja, sa, rhs, solution, add);
}
void LinearSolverFactory::setTolerance(const double tol)

3
opm/core/linalg/LinearSolverFactory.hpp
View File

@@ -74,7 +74,8 @@ namespace Opm
const int* ja,
const double* sa,
const double* rhs,
double* solution) const;
double* solution,
const boost::any& add=boost::any()) const;
/// Set tolerance for the linear solver.
/// \param[in] tol tolerance value

4
opm/core/linalg/LinearSolverInterface.hpp
View File

@@ -20,6 +20,7 @@
#ifndef OPM_LINEARSOLVERINTERFACE_HEADER_INCLUDED
#define OPM_LINEARSOLVERINTERFACE_HEADER_INCLUDED
#include<boost/any.hpp>
struct CSRMatrix;
@@ -69,7 +70,8 @@ namespace Opm
const int* ja,
const double* sa,
const double* rhs,
double* solution) const = 0;
double* solution,
const boost::any& add=boost::any()) const = 0;
/// Set tolerance for the linear solver.
/// \param[in] tol tolerance value

96
opm/core/linalg/LinearSolverIstl.cpp
View File

@@ -23,6 +23,8 @@
#endif
#include <opm/core/linalg/LinearSolverIstl.hpp>
#include <opm/core/linalg/ParallelIstlInformation.hpp>
#include <opm/core/utility/ErrorMacros.hpp>
// Silence compatibility warning from DUNE headers since we don't use
// the deprecated member anyway (in this compilation unit)
@@ -35,7 +37,9 @@
#include <dune/istl/bcrsmatrix.hh>
#include <dune/istl/operators.hh>
#include <dune/istl/io.hh>
#include <dune/istl/owneroverlapcopy.hh>
#include <dune/istl/preconditioners.hh>
#include <dune/istl/schwarz.hh>
#include <dune/istl/solvers.hh>
#include <dune/istl/paamg/amg.hh>
#include <dune/istl/paamg/kamg.hh>
@@ -47,7 +51,7 @@
#include <stdexcept>
#include <iostream>
#include <type_traits>
namespace Opm
{
@@ -134,7 +138,8 @@ namespace Opm
const int* ja,
const double* sa,
const double* rhs,
double* solution) const
double* solution,
const boost::any& comm) const
{
// Build Istl structures from input.
// System matrix
@@ -155,10 +160,26 @@ namespace Opm
if (maxit == 0) {
maxit = 5000;
}
Dune::SeqScalarProduct<Vector> sp;
Dune::Amg::SequentialInformation comm;
Operator opA(A);
return solveSystem(opA, solution, rhs, sp, comm, maxit);
#if HAVE_MPI
if(comm.type()==typeid(ParallelISTLInformation))
{
typedef Dune::OwnerOverlapCopyCommunication<int,int> Comm;
const ParallelISTLInformation& info = boost::any_cast<const ParallelISTLInformation&>(comm);
Comm istlComm(info.communicator());
info.copyValuesTo(istlComm.indexSet(), istlComm.remoteIndices());
Dune::OverlappingSchwarzOperator<Mat,Vector,Vector, Comm>
opA(A, istlComm);
Dune::OverlappingSchwarzScalarProduct<Vector,Comm> sp(istlComm);
return solveSystem(opA, solution, rhs, sp, istlComm, maxit);
}
else
#endif
{
Dune::SeqScalarProduct<Vector> sp;
Dune::Amg::SequentialInformation comm;
Operator opA(A);
return solveSystem(opA, solution, rhs, sp, comm, maxit);
}
}
template<class O, class S, class C>
@@ -204,6 +225,14 @@ namespace Opm
break;
case FastAMG:
#if defined(HAS_DUNE_FAST_AMG) || DUNE_VERSION_NEWER(DUNE_ISTL, 2, 3)
#if HAVE_MPI
if(std::is_same<C,Dune::OwnerOverlapCopyCommunication<int,int> >::value)
{
OPM_THROW(std::runtime_error, "Trying to use sequential FastAMG solver for a parallel problem!");
}
#endif // HAVE_MPI
res = solveFastAMG(opA, x, b, sp, comm, linsolver_residual_tolerance_, maxit, linsolver_verbosity_,
linsolver_prolongate_factor_);
#else
@@ -234,28 +263,47 @@ namespace Opm
return linsolver_residual_tolerance_;
}
namespace
{
template<class P, class O, class C>
struct SmootherChooser
{
typedef P Type;
};
#if HAVE_MPI
template<class P, class O>
struct SmootherChooser<P, O, Dune::OwnerOverlapCopyCommunication<int,int> >
{
typedef Dune::OwnerOverlapCopyCommunication<int,int> Comm;
typedef Dune::BlockPreconditioner<typename O::domain_type, typename O::range_type,
Comm, P>
Type;
};
#endif
template<class P, class O, class C>
struct PreconditionerTraits
{
typedef std::shared_ptr<P> PointerType;
typedef typename SmootherChooser<P,O,C>::Type SmootherType;
typedef std::shared_ptr<SmootherType> PointerType;
};
template<class P, class O, class C>
typename PreconditionerTraits<P,O>::PointerType
typename PreconditionerTraits<P,O,C>::PointerType
makePreconditioner(O& opA, double relax, const C& comm, int iterations=1)
{
typename Dune::Amg::SmootherTraits<P>::Arguments args;
typename Dune::Amg::ConstructionTraits<P>::Arguments cargs;
typedef typename SmootherChooser<P,O,C>::Type SmootherType;
typedef typename PreconditionerTraits<P,O,C>::PointerType PointerType;
typename Dune::Amg::SmootherTraits<SmootherType>::Arguments args;
typename Dune::Amg::ConstructionTraits<SmootherType>::Arguments cargs;
cargs.setMatrix(opA.getmat());
args.iterations=iterations;
args.relaxationFactor=relax;
cargs.setArgs(args);
cargs.setComm(comm);
return std::shared_ptr<P>(Dune::Amg::ConstructionTraits<P>::construct(cargs));
return PointerType(Dune::Amg::ConstructionTraits<SmootherType>::construct(cargs));
}
template<class O, class S, class C>
@@ -323,19 +371,20 @@ namespace Opm
#endif
#if SMOOTHER_ILU
typedef Dune::SeqILU0<Mat,Vector,Vector> Smoother;
typedef Dune::SeqILU0<Mat,Vector,Vector> SeqSmoother;
#else
typedef Dune::SeqSOR<Mat,Vector,Vector> Smoother;
typedef Dune::SeqSOR<Mat,Vector,Vector> SeqSmoother;
#endif
typedef typename SmootherChooser<SeqSmoother, O, C>::Type Smoother;
typedef Dune::Amg::CoarsenCriterion<CriterionBase> Criterion;
typedef Dune::Amg::AMG<Operator,Vector,Smoother> Precond;
typedef Dune::Amg::AMG<O,Vector,Smoother,C> Precond;
// Construct preconditioner.
Criterion criterion;
Precond::SmootherArgs smootherArgs;
typename Precond::SmootherArgs smootherArgs;
setUpCriterion(criterion, linsolver_prolongate_factor, verbosity,
linsolver_smooth_steps);
Precond precond(opA, criterion, smootherArgs);
Precond precond(opA, criterion, smootherArgs, comm);
// Construct linear solver.
Dune::CGSolver<Vector> linsolve(opA, sp, precond, tolerance, maxit, verbosity);
@@ -360,6 +409,7 @@ namespace Opm
double linsolver_prolongate_factor, int linsolver_smooth_steps)
{
// Solve with AMG solver.
Dune::MatrixAdapter<typename O::matrix_type,Vector,Vector> sOpA(opA.getmat());
#if FIRST_DIAGONAL
typedef Dune::Amg::FirstDiagonal CouplingMetric;
@@ -386,10 +436,10 @@ namespace Opm
Criterion criterion;
setUpCriterion(criterion, linsolver_prolongate_factor, verbosity,
linsolver_smooth_steps);
Precond precond(opA, criterion, smootherArgs);
Precond precond(sOpA, criterion, smootherArgs);
// Construct linear solver.
Dune::GeneralizedPCGSolver<Vector> linsolve(opA, sp, precond, tolerance, maxit, verbosity);
Dune::GeneralizedPCGSolver<Vector> linsolve(sOpA, precond, tolerance, maxit, verbosity);
// Solve system.
Dune::InverseOperatorResult result;
@@ -409,6 +459,8 @@ namespace Opm
double linsolver_prolongate_factor)
{
// Solve with AMG solver.
typedef Dune::MatrixAdapter<typename O::matrix_type, Vector, Vector> Operator;
Operator sOpA(opA.getmat());
#if FIRST_DIAGONAL
typedef Dune::Amg::FirstDiagonal CouplingMetric;
@@ -434,10 +486,10 @@ namespace Opm
parms.setNoPreSmoothSteps(smooth_steps);
parms.setNoPostSmoothSteps(smooth_steps);
parms.setProlongationDampingFactor(linsolver_prolongate_factor);
Precond precond(opA, criterion, parms);
Precond precond(sOpA, criterion, parms);
// Construct linear solver.
Dune::GeneralizedPCGSolver<Vector> linsolve(opA, sp, precond, tolerance, maxit, verbosity);
Dune::GeneralizedPCGSolver<Vector> linsolve(sOpA, precond, tolerance, maxit, verbosity);
// Solve system.
Dune::InverseOperatorResult result;

6
opm/core/linalg/LinearSolverIstl.hpp
View File

@@ -24,12 +24,11 @@
#include <opm/core/linalg/LinearSolverInterface.hpp>
#include <opm/core/utility/parameters/ParameterGroup.hpp>
#include <string>
#include <boost/any.hpp>
namespace Opm
{
/// Concrete class encapsulating some dune-istl linear solvers.
class LinearSolverIstl : public LinearSolverInterface
{
@@ -75,7 +74,8 @@ namespace Opm
const int* ja,
const double* sa,
const double* rhs,
double* solution) const;
double* solution,
const boost::any& comm=boost::any()) const;
/// Set tolerance for the residual in dune istl linear solver.
/// \param[in] tol tolerance value

3
opm/core/linalg/LinearSolverUmfpack.cpp
View File

@@ -46,7 +46,8 @@ namespace Opm
const int* ja,
const double* sa,
const double* rhs,
double* solution) const
double* solution,
const boost::any&) const
{
CSRMatrix A = {
(size_t)size,

3
opm/core/linalg/LinearSolverUmfpack.hpp
View File

@@ -55,7 +55,8 @@ namespace Opm
const int* ja,
const double* sa,
const double* rhs,
double* solution) const;
double* solution,
const boost::any& add=boost::any()) const;
/// Set tolerance for the linear solver.
/// \param[in] tol tolerance value

160
opm/core/linalg/ParallelIstlInformation.hpp Normal file
View File

@@ -0,0 +1,160 @@
/*
Copyright 2014 Dr. Markus Blatt - HPC-Simulation-Software & Services
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPM_PARALLELISTLINFORMTION_HEADER_INCLUDED
#define OPM_PARALLELISTLINFORMTION_HEADER_INCLUDED
#if HAVE_MPI && HAVE_DUNE_ISTL
#include "mpi.h"
#include <dune/istl/owneroverlapcopy.hh>
#include <dune/common/parallel/interface.hh>
#include <dune/common/parallel/communicator.hh>
#include <dune/common/enumset.hh>
#include<algorithm>
namespace Opm
{
/// \brief Class that encapsulates the parallelization information needed by the
/// ISTL solvers.
class ParallelISTLInformation
{
public:
/// \brief The type of the parallel index set used.
typedef Dune::OwnerOverlapCopyCommunication<int, int>::ParallelIndexSet ParallelIndexSet;
/// \brief The type of the remote indices information used.
typedef Dune::OwnerOverlapCopyCommunication<int, int>::RemoteIndices RemoteIndices;
/// \brief Constructs an empty parallel information object using MPI_COMM_WORLD
ParallelISTLInformation()
: indexSet_(new ParallelIndexSet),
remoteIndices_(new RemoteIndices(*indexSet_, *indexSet_, MPI_COMM_WORLD)),
communicator_(MPI_COMM_WORLD)
{}
/// \brief Constructs an empty parallel information object using a communicator.
/// \param communicator The communicator to use.
ParallelISTLInformation(MPI_Comm communicator)
: indexSet_(new ParallelIndexSet),
remoteIndices_(new RemoteIndices(*indexSet_, *indexSet_, communicator)),
communicator_(communicator)
{}
/// \brief Constructs a parallel information object from the specified information.
/// \param indexSet The parallel index set to use.
/// \param remoteIndices The remote indices information to use.
/// \param communicator The communicator to use.
ParallelISTLInformation(const std::shared_ptr<ParallelIndexSet>& indexSet,
const std::shared_ptr<RemoteIndices>& remoteIndices,
MPI_Comm communicator)
: indexSet_(indexSet), remoteIndices_(remoteIndices), communicator_(communicator)
{}
/// \brief Copy constructor.
///
/// The information will be shared by the the two objects.
ParallelISTLInformation(const ParallelISTLInformation& other)
: indexSet_(other.indexSet_), remoteIndices_(other.remoteIndices_),
communicator_(other.communicator_)
{}
/// \brief Get a pointer to the underlying index set.
std::shared_ptr<ParallelIndexSet> indexSet() const
{
return indexSet_;
}
/// \brief Get a pointer to the remote indices information.
std::shared_ptr<RemoteIndices> remoteIndices() const
{
return remoteIndices_;
}
/// \brief Get the MPI communicator that we use.
MPI_Comm communicator() const
{
return communicator_;
}
/// \brief Copy the information stored to the specified objects.
/// \param[out] indexSet The object to store the index set in.
/// \param[out] remoteIndices The object to store the remote indices information in.
void copyValuesTo(ParallelIndexSet& indexSet, RemoteIndices& remoteIndices) const
{
indexSet.beginResize();
IndexSetInserter<ParallelIndexSet> inserter(indexSet);
std::for_each(indexSet_->begin(), indexSet_->end(), inserter);
indexSet.endResize();
remoteIndices.rebuild<false>();
}
/// \brief Communcate the dofs owned by us to the other process.
///
/// Afterwards all associated dofs will contain the same data.
template<class T>
void copyOwnerToAll (const T& source, T& dest) const
{
typedef Dune::Combine<Dune::EnumItem<Dune::OwnerOverlapCopyAttributeSet::AttributeSet,Dune::OwnerOverlapCopyAttributeSet::owner>,Dune::EnumItem<Dune::OwnerOverlapCopyAttributeSet::AttributeSet,Dune::OwnerOverlapCopyAttributeSet::overlap>,Dune::OwnerOverlapCopyAttributeSet::AttributeSet> OwnerOverlapSet;
typedef Dune::Combine<OwnerOverlapSet, Dune::EnumItem<Dune::OwnerOverlapCopyAttributeSet::AttributeSet,Dune::OwnerOverlapCopyAttributeSet::copy>,Dune::OwnerOverlapCopyAttributeSet::AttributeSet> AllSet;
OwnerOverlapSet sourceFlags;
AllSet destFlags;
Dune::Interface interface(communicator_);
if(!remoteIndices_->isSynced())
remoteIndices_->rebuild<false>();
interface.build(*remoteIndices_,sourceFlags,destFlags);
Dune::BufferedCommunicator communicator;
communicator.template build<T>(interface);
communicator.template forward<CopyGatherScatter<T> >(source,dest);
communicator.free();
}
private:
/** \brief gather/scatter callback for communcation */
template<typename T>
struct CopyGatherScatter
{
typedef typename Dune::CommPolicy<T>::IndexedType V;
static V gather(const T& a, std::size_t i)
{
return a[i];
}
static void scatter(T& a, V v, std::size_t i)
{
a[i] = v;
}
};
template<class T>
class IndexSetInserter
{
public:
typedef T ParallelIndexSet;
IndexSetInserter(ParallelIndexSet& indexSet)
: indexSet_(&indexSet)
{}
void operator()(const typename ParallelIndexSet::IndexPair& pair)
{
indexSet_->add(pair.global(), pair.local());
}
private:
ParallelIndexSet* indexSet_;
};
std::shared_ptr<ParallelIndexSet> indexSet_;
std::shared_ptr<RemoteIndices> remoteIndices_;
MPI_Comm communicator_;
};
} // end namespace Opm
#endif
#endif