diff --git a/opm/simulators/linalg/FlowLinearSolverParameters.hpp b/opm/simulators/linalg/FlowLinearSolverParameters.hpp index fff4b467e..3feb92c88 100644 --- a/opm/simulators/linalg/FlowLinearSolverParameters.hpp +++ b/opm/simulators/linalg/FlowLinearSolverParameters.hpp @@ -48,155 +48,6 @@ struct FlowIstlSolverParams {}; } -template -struct LinearSolverReduction { using type = UndefinedProperty; }; - -template -struct RelaxedLinearSolverReduction { using type = UndefinedProperty; }; - -template -struct LinearSolverMaxIter { using type = UndefinedProperty; }; - -template -struct LinearSolverRestart { using type = UndefinedProperty; }; - -template -struct IluRelaxation { using type = UndefinedProperty; }; - -template -struct IluFillinLevel { using type = UndefinedProperty; }; - -template -struct MiluVariant { using type = UndefinedProperty; }; - -template -struct IluRedblack { using type = UndefinedProperty; }; - -template -struct IluReorderSpheres { using type = UndefinedProperty; }; - -template -struct UseGmres { using type = UndefinedProperty; }; - -template -struct LinearSolverIgnoreConvergenceFailure { using type = UndefinedProperty; }; - -template -struct ScaleLinearSystem { using type = UndefinedProperty; }; - -template -struct LinearSolver { using type = UndefinedProperty; }; - -template -struct LinearSolverPrintJsonDefinition { using type = UndefinedProperty; }; - -template -struct CprReuseSetup { using type = UndefinedProperty; }; - -template -struct CprReuseInterval { using type = UndefinedProperty; }; - -template -struct AcceleratorMode { using type = UndefinedProperty; }; - -template -struct BdaDeviceId { using type = UndefinedProperty; }; - -template -struct OpenclPlatformId { using type = UndefinedProperty; }; - -template -struct OpenclIluParallel { using type = UndefinedProperty; }; - -template -struct LinearSolverReduction -{ - using type = GetPropType; - static constexpr type value = 1e-2; -}; - -template -struct RelaxedLinearSolverReduction -{ - using type = GetPropType; - static constexpr type value = 1e-2; -}; - -template -struct LinearSolverMaxIter -{ static constexpr int value = 200; }; - -template -struct LinearSolverRestart -{ static constexpr int value = 40; }; - -template -struct IluRelaxation -{ - using type = GetPropType; - static constexpr type value = 0.9; -}; - -template -struct IluFillinLevel -{ static constexpr int value = 0; }; - -template -struct MiluVariant -{ static constexpr auto value = "ILU"; }; - -template -struct IluRedblack -{ static constexpr bool value = false; }; - -template -struct IluReorderSpheres -{ static constexpr bool value = false; }; - -template -struct UseGmres -{ static constexpr bool value = false; }; - -template -struct LinearSolverIgnoreConvergenceFailure -{ static constexpr bool value = false; }; - -template -struct ScaleLinearSystem -{ static constexpr bool value = false; }; - -template -struct LinearSolver -{ static constexpr auto value = "ilu0"; }; - -template -struct LinearSolverPrintJsonDefinition -{ static constexpr auto value = true; }; - -template -struct CprReuseSetup -{ static constexpr int value = 4; }; - -template -struct CprReuseInterval -{ static constexpr int value = 30; }; - -template -struct AcceleratorMode -{ static constexpr auto value = "none"; }; - -template -struct BdaDeviceId -{ static constexpr int value = 0; }; - -template -struct OpenclPlatformId -{ static constexpr int value = 0; }; - -template -struct OpenclIluParallel -{ static constexpr bool value = true; }; // note: false should only be used in debug - // Set the backend to be used. template struct LinearSolverBackend @@ -208,15 +59,164 @@ struct LinearSolverBackend #endif }; -} // namespace Opm::Properties +} namespace Opm::Parameters { +template +struct LinearSolverReduction { using type = Properties::UndefinedProperty; }; + +template +struct RelaxedLinearSolverReduction { using type = Properties::UndefinedProperty; }; + +template +struct LinearSolverMaxIter { using type = Properties::UndefinedProperty; }; + +template +struct LinearSolverRestart { using type = Properties::UndefinedProperty; }; + +template +struct IluRelaxation { using type = Properties::UndefinedProperty; }; + +template +struct IluFillinLevel { using type = Properties::UndefinedProperty; }; + +template +struct MiluVariant { using type = Properties::UndefinedProperty; }; + +template +struct IluRedblack { using type = Properties::UndefinedProperty; }; + +template +struct IluReorderSpheres { using type = Properties::UndefinedProperty; }; + +template +struct UseGmres { using type = Properties::UndefinedProperty; }; + +template +struct LinearSolverIgnoreConvergenceFailure { using type = Properties::UndefinedProperty; }; + +template +struct ScaleLinearSystem { using type = Properties::UndefinedProperty; }; + +template +struct LinearSolver { using type = Properties::UndefinedProperty; }; + +template +struct LinearSolverPrintJsonDefinition { using type = Properties::UndefinedProperty; }; + +template +struct CprReuseSetup { using type = Properties::UndefinedProperty; }; + +template +struct CprReuseInterval { using type = Properties::UndefinedProperty; }; + +template +struct AcceleratorMode { using type = Properties::UndefinedProperty; }; + +template +struct BdaDeviceId { using type = Properties::UndefinedProperty; }; + +template +struct OpenclPlatformId { using type = Properties::UndefinedProperty; }; + +template +struct OpenclIluParallel { using type = Properties::UndefinedProperty; }; + +template +struct LinearSolverReduction +{ + using type = GetPropType; + static constexpr type value = 1e-2; +}; + +template +struct RelaxedLinearSolverReduction +{ + using type = GetPropType; + static constexpr type value = 1e-2; +}; + +template +struct LinearSolverMaxIter +{ static constexpr int value = 200; }; + +template +struct LinearSolverRestart +{ static constexpr int value = 40; }; + +template +struct IluRelaxation +{ + using type = GetPropType; + static constexpr type value = 0.9; +}; + +template +struct IluFillinLevel +{ static constexpr int value = 0; }; + +template +struct MiluVariant +{ static constexpr auto value = "ILU"; }; + +template +struct IluRedblack +{ static constexpr bool value = false; }; + +template +struct IluReorderSpheres +{ static constexpr bool value = false; }; + +template +struct UseGmres +{ static constexpr bool value = false; }; + +template +struct LinearSolverIgnoreConvergenceFailure +{ static constexpr bool value = false; }; + +template +struct ScaleLinearSystem +{ static constexpr bool value = false; }; + +template +struct LinearSolver +{ static constexpr auto value = "ilu0"; }; + +template +struct LinearSolverPrintJsonDefinition +{ static constexpr auto value = true; }; + +template +struct CprReuseSetup +{ static constexpr int value = 4; }; + +template +struct CprReuseInterval +{ static constexpr int value = 30; }; + +template +struct AcceleratorMode +{ static constexpr auto value = "none"; }; + +template +struct BdaDeviceId +{ static constexpr int value = 0; }; + +template +struct OpenclPlatformId +{ static constexpr int value = 0; }; + +template +struct OpenclIluParallel +{ static constexpr bool value = true; }; // note: false should only be used in debug + template struct LinearSolverVerbosity { static constexpr int value = 0; }; -} // namespace Opm::Properties +} // namespace Opm::Parameters namespace Opm { @@ -249,105 +249,105 @@ struct FlowLinearSolverParameters void init(bool cprRequestedInDataFile) { // TODO: these parameters have undocumented non-trivial dependencies - linear_solver_reduction_ = Parameters::get(); - relaxed_linear_solver_reduction_ = Parameters::get(); - linear_solver_maxiter_ = Parameters::get(); - linear_solver_restart_ = Parameters::get(); + linear_solver_reduction_ = Parameters::get(); + relaxed_linear_solver_reduction_ = Parameters::get(); + linear_solver_maxiter_ = Parameters::get(); + linear_solver_restart_ = Parameters::get(); linear_solver_verbosity_ = Parameters::get(); - ilu_relaxation_ = Parameters::get(); - ilu_fillin_level_ = Parameters::get(); - ilu_milu_ = convertString2Milu(Parameters::get()); - ilu_redblack_ = Parameters::get(); - ilu_reorder_sphere_ = Parameters::get(); - newton_use_gmres_ = Parameters::get(); - ignoreConvergenceFailure_ = Parameters::get(); - scale_linear_system_ = Parameters::get(); - linsolver_ = Parameters::get(); - linear_solver_print_json_definition_ = Parameters::get(); - cpr_reuse_setup_ = Parameters::get(); - cpr_reuse_interval_ = Parameters::get(); + ilu_relaxation_ = Parameters::get(); + ilu_fillin_level_ = Parameters::get(); + ilu_milu_ = convertString2Milu(Parameters::get()); + ilu_redblack_ = Parameters::get(); + ilu_reorder_sphere_ = Parameters::get(); + newton_use_gmres_ = Parameters::get(); + ignoreConvergenceFailure_ = Parameters::get(); + scale_linear_system_ = Parameters::get(); + linsolver_ = Parameters::get(); + linear_solver_print_json_definition_ = Parameters::get(); + cpr_reuse_setup_ = Parameters::get(); + cpr_reuse_interval_ = Parameters::get(); - if (!Parameters::isSet() && cprRequestedInDataFile) { + if (!Parameters::isSet() && cprRequestedInDataFile) { linsolver_ = "cpr"; } else { - linsolver_ = Parameters::get(); + linsolver_ = Parameters::get(); } - accelerator_mode_ = Parameters::get(); - bda_device_id_ = Parameters::get(); - opencl_platform_id_ = Parameters::get(); - opencl_ilu_parallel_ = Parameters::get(); + accelerator_mode_ = Parameters::get(); + bda_device_id_ = Parameters::get(); + opencl_platform_id_ = Parameters::get(); + opencl_ilu_parallel_ = Parameters::get(); } template static void registerParameters() { - Parameters::registerParam + Parameters::registerParam ("The minimum reduction of the residual which the linear solver must achieve"); - Parameters::registerParam + Parameters::registerParam ("The minimum reduction of the residual which the linear solver need to " "achieve for the solution to be accepted"); - Parameters::registerParam + Parameters::registerParam ("The maximum number of iterations of the linear solver"); - Parameters::registerParam + Parameters::registerParam ("The number of iterations after which GMRES is restarted"); Parameters::registerParam ("The verbosity level of the linear solver (0: off, 2: all)"); - Parameters::registerParam + Parameters::registerParam ("The relaxation factor of the linear solver's ILU preconditioner"); - Parameters::registerParam + Parameters::registerParam ("The fill-in level of the linear solver's ILU preconditioner"); - Parameters::registerParam + Parameters::registerParam ("Specify which variant of the modified-ILU preconditioner ought to be used. " "Possible variants are: ILU (default, plain ILU), " "MILU_1 (lump diagonal with dropped row entries), " "MILU_2 (lump diagonal with the sum of the absolute values of the dropped row entries), " "MILU_3 (if diagonal is positive add sum of dropped row entries, otherwise subtract them), " "MILU_4 (if diagonal is positive add sum of dropped row entries, otherwise do nothing"); - Parameters::registerParam + Parameters::registerParam ("Use red-black partitioning for the ILU preconditioner"); - Parameters::registerParam + Parameters::registerParam ("Whether to reorder the entries of the matrix in the red-black " "ILU preconditioner in spheres starting at an edge. " "If false the original ordering is preserved in each color. " "Otherwise why try to ensure D4 ordering (in a 2D structured grid, " "the diagonal elements are consecutive)."); - Parameters::registerParam + Parameters::registerParam ("Use GMRES as the linear solver"); - Parameters::registerParam + Parameters::registerParam ("Continue with the simulation like nothing happened " "after the linear solver did not converge"); - Parameters::registerParam + Parameters::registerParam ("Scale linear system according to equation scale and primary variable types"); - Parameters::registerParam + Parameters::registerParam ("Configuration of solver. Valid options are: ilu0 (default), " "dilu, cprw, cpr (an alias for cprw), cpr_quasiimpes, " "cpr_trueimpes, cpr_trueimpesanalytic, amg or hybrid (experimental). " "Alternatively, you can request a configuration to be read from a " "JSON file by giving the filename here, ending with '.json.'"); - Parameters::registerParam + Parameters::registerParam ("Write the JSON definition of the linear solver setup to the DBG file."); - Parameters::registerParam + Parameters::registerParam ("Reuse preconditioner setup. Valid options are " "0: recreate the preconditioner for every linear solve, " "1: recreate once every timestep, " "2: recreate if last linear solve took more than 10 iterations, " "3: never recreate, " "4: recreated every CprReuseInterval"); - Parameters::registerParam + Parameters::registerParam ("Reuse preconditioner interval. Used when CprReuseSetup is set to 4, " "then the preconditioner will be fully recreated instead of reused " "every N linear solve, where N is this parameter."); - Parameters::registerParam + Parameters::registerParam ("Choose a linear solver, usage: " "'--accelerator-mode=[none|cusparse|opencl|amgcl|rocalution|rocsparse]'"); - Parameters::registerParam + Parameters::registerParam ("Choose device ID for cusparseSolver or openclSolver, " "use 'nvidia-smi' or 'clinfo' to determine valid IDs"); - Parameters::registerParam + Parameters::registerParam ("Choose platform ID for openclSolver, use 'clinfo' " "to determine valid platform IDs"); - Parameters::registerParam + Parameters::registerParam ("Parallelize ILU decomposition and application on GPU"); } diff --git a/opm/simulators/linalg/ISTLSolver.hpp b/opm/simulators/linalg/ISTLSolver.hpp index 2c92c3cb6..7eddf6339 100644 --- a/opm/simulators/linalg/ISTLSolver.hpp +++ b/opm/simulators/linalg/ISTLSolver.hpp @@ -224,8 +224,8 @@ std::unique_ptr blockJacobiAdjacency(const Grid& grid, para.linsolver_ = "cprw"; parameters_.push_back(para); prm_.push_back(setupPropertyTree(parameters_[0], - Parameters::isSet(), - Parameters::isSet())); + Parameters::isSet(), + Parameters::isSet())); } { FlowLinearSolverParameters para; @@ -233,8 +233,8 @@ std::unique_ptr blockJacobiAdjacency(const Grid& grid, para.linsolver_ = "ilu0"; parameters_.push_back(para); prm_.push_back(setupPropertyTree(parameters_[1], - Parameters::isSet(), - Parameters::isSet())); + Parameters::isSet(), + Parameters::isSet())); } // ------------ } else { @@ -242,8 +242,8 @@ std::unique_ptr blockJacobiAdjacency(const Grid& grid, assert(parameters_.size() == 1); assert(prm_.empty()); prm_.push_back(setupPropertyTree(parameters_[0], - Parameters::isSet(), - Parameters::isSet())); + Parameters::isSet(), + Parameters::isSet())); } flexibleSolver_.resize(prm_.size()); diff --git a/opm/simulators/linalg/ISTLSolverBda.hpp b/opm/simulators/linalg/ISTLSolverBda.hpp index 8a94d966a..d977b4f8e 100644 --- a/opm/simulators/linalg/ISTLSolverBda.hpp +++ b/opm/simulators/linalg/ISTLSolverBda.hpp @@ -152,7 +152,7 @@ public: { OPM_TIMEBLOCK(initializeBda); - std::string accelerator_mode = Parameters::get(); + std::string accelerator_mode = Parameters::get(); // Force accelerator mode to none if using MPI. if ((this->simulator_.vanguard().grid().comm().size() > 1) && (accelerator_mode != "none")) { const bool on_io_rank = (this->simulator_.gridView().comm().rank() == 0); @@ -167,13 +167,13 @@ public: } // Initialize the BdaBridge - const int platformID = Parameters::get(); - const int deviceID = Parameters::get(); - const int maxit = Parameters::get(); - const double tolerance = Parameters::get(); - const bool opencl_ilu_parallel = Parameters::get(); + const int platformID = Parameters::get(); + const int deviceID = Parameters::get(); + const int maxit = Parameters::get(); + const double tolerance = Parameters::get(); + const bool opencl_ilu_parallel = Parameters::get(); const int linear_solver_verbosity = this->parameters_[0].linear_solver_verbosity_; - std::string linsolver = Parameters::get(); + std::string linsolver = Parameters::get(); bdaBridge_ = std::make_unique>(accelerator_mode, linear_solver_verbosity, maxit,