From d25405d39afd4185b0d91254cb03da58055974d2 Mon Sep 17 00:00:00 2001 From: tqiu Date: Fri, 16 Oct 2020 15:05:02 +0200 Subject: [PATCH] Renamed parameter to OpenclIluReorder and moved error checking to BdaBridge constructor. --- CMakeLists_files.cmake | 1 + .../linalg/FlowLinearSolverParameters.hpp | 12 +++---- opm/simulators/linalg/bda/BILU0.cpp | 20 ++++------- opm/simulators/linalg/bda/BILU0.hpp | 7 ++-- opm/simulators/linalg/bda/BdaBridge.cpp | 15 ++++++-- opm/simulators/linalg/bda/BdaBridge.hpp | 6 ++-- opm/simulators/linalg/bda/ILUReorder.hpp | 35 +++++++++++++++++++ .../linalg/bda/openclSolverBackend.cpp | 6 ++-- .../linalg/bda/openclSolverBackend.hpp | 5 +-- 9 files changed, 73 insertions(+), 34 deletions(-) create mode 100644 opm/simulators/linalg/bda/ILUReorder.hpp diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index 380613f59..3e65d6f42 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -58,6 +58,7 @@ if(OPENCL_FOUND) list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/BlockedMatrix.cpp) list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/BILU0.cpp) list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/Reorder.cpp) + list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/ILUReorder.cpp) list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/opencl.cpp) list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/openclSolverBackend.cpp) list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/BdaBridge.cpp) diff --git a/opm/simulators/linalg/FlowLinearSolverParameters.hpp b/opm/simulators/linalg/FlowLinearSolverParameters.hpp index cb1912e20..ae14f9ac0 100644 --- a/opm/simulators/linalg/FlowLinearSolverParameters.hpp +++ b/opm/simulators/linalg/FlowLinearSolverParameters.hpp @@ -130,7 +130,7 @@ struct OpenclPlatformId { using type = UndefinedProperty; }; template -struct IluReorderStrategy { +struct OpenclIluReorder { using type = UndefinedProperty; }; @@ -225,7 +225,7 @@ struct OpenclPlatformId { static constexpr int value = 0; }; template -struct IluReorderStrategy { +struct OpenclIluReorder { static constexpr auto value = "graph_coloring"; }; @@ -257,7 +257,7 @@ namespace Opm int opencl_platform_id_; int cpr_max_ell_iter_ = 20; int cpr_reuse_setup_ = 0; - std::string ilu_reorder_strategy_; + std::string opencl_ilu_reorder_; template void init() @@ -282,7 +282,7 @@ namespace Opm gpu_mode_ = EWOMS_GET_PARAM(TypeTag, std::string, GpuMode); bda_device_id_ = EWOMS_GET_PARAM(TypeTag, int, BdaDeviceId); opencl_platform_id_ = EWOMS_GET_PARAM(TypeTag, int, OpenclPlatformId); - ilu_reorder_strategy_ = EWOMS_GET_PARAM(TypeTag, std::string, IluReorderStrategy); + opencl_ilu_reorder_ = EWOMS_GET_PARAM(TypeTag, std::string, OpenclIluReorder); } template @@ -307,7 +307,7 @@ namespace Opm EWOMS_REGISTER_PARAM(TypeTag, std::string, GpuMode, "Use GPU cusparseSolver or openclSolver as the linear solver, usage: '--gpu-mode=[none|cusparse|opencl]'"); EWOMS_REGISTER_PARAM(TypeTag, int, BdaDeviceId, "Choose device ID for cusparseSolver or openclSolver, use 'nvidia-smi' or 'clinfo' to determine valid IDs"); EWOMS_REGISTER_PARAM(TypeTag, int, OpenclPlatformId, "Choose platform ID for openclSolver, use 'clinfo' to determine valid platform IDs"); - EWOMS_REGISTER_PARAM(TypeTag, std::string, IluReorderStrategy, "Choose the reordering strategy for openclSolver, usage: '--ilu-reorder-strategy=[level_scheduling|graph_coloring], level_scheduling behaves like Dune and cusparse, graph_coloring is more aggressive and likely to be faster, but is random-based and generally increases the number of linear solves and linear iterations significantly."); + EWOMS_REGISTER_PARAM(TypeTag, std::string, OpenclIluReorder, "Choose the reordering strategy for ILU for openclSolver, usage: '--opencl-ilu-reorder=[level_scheduling|graph_coloring], level_scheduling behaves like Dune and cusparse, graph_coloring is more aggressive and likely to be faster, but is random-based and generally increases the number of linear solves and linear iterations significantly."); } FlowLinearSolverParameters() { reset(); } @@ -330,7 +330,7 @@ namespace Opm gpu_mode_ = "none"; bda_device_id_ = 0; opencl_platform_id_ = 0; - ilu_reorder_strategy_ = "graph_coloring"; + opencl_ilu_reorder_ = "graph_coloring"; } }; diff --git a/opm/simulators/linalg/bda/BILU0.cpp b/opm/simulators/linalg/bda/BILU0.cpp index 70139a429..24905520d 100644 --- a/opm/simulators/linalg/bda/BILU0.cpp +++ b/opm/simulators/linalg/bda/BILU0.cpp @@ -35,17 +35,9 @@ namespace bda using Dune::Timer; template - BILU0::BILU0(std::string ilu_reorder_strategy, int verbosity_) : - verbosity(verbosity_) - { - if (ilu_reorder_strategy.compare("level_scheduling") == 0) { - level_scheduling = true; - } else if (ilu_reorder_strategy.compare("graph_coloring") == 0) { - graph_coloring = true; - } else { - OPM_THROW(std::logic_error, "Error invalid argument for --ilu-reorder-strategy, usage: '--ilu-reorder-strategy=[level_scheduling|graph_coloring]'"); - } - } + BILU0::BILU0(ILUReorder opencl_ilu_reorder_, int verbosity_) : + verbosity(verbosity_), opencl_ilu_reorder(opencl_ilu_reorder_) + {} template BILU0::~BILU0() @@ -84,10 +76,10 @@ namespace bda rmat = std::make_shared >(mat->Nb, mat->nnzbs); LUmat = std::make_unique >(*rmat); std::ostringstream out; - if (level_scheduling) { + if (opencl_ilu_reorder == ILUReorder::LEVEL_SCHEDULING) { out << "BILU0 reordering strategy: " << "level_scheduling\n"; findLevelScheduling(mat->colIndices, mat->rowPointers, CSCRowIndices, CSCColPointers, mat->Nb, &numColors, toOrder, fromOrder, rowsPerColor); - } else if (graph_coloring) { + } else if (opencl_ilu_reorder == ILUReorder::GRAPH_COLORING) { out << "BILU0 reordering strategy: " << "graph_coloring\n"; findGraphColoring(mat->colIndices, mat->rowPointers, CSCRowIndices, CSCColPointers, mat->Nb, mat->Nb, mat->Nb, &numColors, toOrder, fromOrder, rowsPerColor); } else { @@ -325,7 +317,7 @@ namespace bda #define INSTANTIATE_BDA_FUNCTIONS(n) \ -template BILU0::BILU0(std::string, int); \ +template BILU0::BILU0(ILUReorder, int); \ template BILU0::~BILU0(); \ template bool BILU0::init(BlockedMatrix*); \ template bool BILU0::create_preconditioner(BlockedMatrix*); \ diff --git a/opm/simulators/linalg/bda/BILU0.hpp b/opm/simulators/linalg/bda/BILU0.hpp index 25bb9c493..bad6201ca 100644 --- a/opm/simulators/linalg/bda/BILU0.hpp +++ b/opm/simulators/linalg/bda/BILU0.hpp @@ -21,6 +21,7 @@ #define BILU0_HPP #include +#include #include @@ -47,9 +48,7 @@ namespace bda int numColors; int verbosity; - // Level Scheduling respects the dependencies in the original matrix, and behaves like Dune and cusparse - // Graph Coloring is more aggresive and is likely to increase the number of linearizations and linear iterations to converge significantly, but can still be faster on GPU because it results in more parallelism - bool level_scheduling = false, graph_coloring = false; + ILUReorder opencl_ilu_reorder; typedef struct { cl::Buffer Lvals, Uvals, invDiagVals; @@ -70,7 +69,7 @@ namespace bda public: - BILU0(std::string ilu_reorder_strategy, int verbosity); + BILU0(ILUReorder opencl_ilu_reorder, int verbosity); ~BILU0(); diff --git a/opm/simulators/linalg/bda/BdaBridge.cpp b/opm/simulators/linalg/bda/BdaBridge.cpp index 150373324..39051dcb0 100644 --- a/opm/simulators/linalg/bda/BdaBridge.cpp +++ b/opm/simulators/linalg/bda/BdaBridge.cpp @@ -38,9 +38,10 @@ namespace Opm using bda::BdaResult; using bda::BdaSolver; using bda::SolverStatus; + using bda::ILUReorder; template -BdaBridge::BdaBridge(std::string gpu_mode, int linear_solver_verbosity, int maxit, double tolerance, unsigned int platformID OPM_UNUSED, unsigned int deviceID, std::string ilu_reorder_strategy OPM_UNUSED) +BdaBridge::BdaBridge(std::string gpu_mode, int linear_solver_verbosity, int maxit, double tolerance, unsigned int platformID OPM_UNUSED, unsigned int deviceID, std::string opencl_ilu_reorder OPM_UNUSED) { if (gpu_mode.compare("cusparse") == 0) { #if HAVE_CUDA @@ -52,7 +53,15 @@ BdaBridge::BdaBridge(std::string gpu_mod } else if (gpu_mode.compare("opencl") == 0) { #if HAVE_OPENCL use_gpu = true; - backend.reset(new bda::openclSolverBackend(linear_solver_verbosity, maxit, tolerance, platformID, deviceID, ilu_reorder_strategy)); + ILUReorder ilu_reorder = bda::ILUReorder::GRAPH_COLORING; + if (opencl_ilu_reorder == "level_scheduling") { + ilu_reorder = bda::ILUReorder::LEVEL_SCHEDULING; + } else if (opencl_ilu_reorder == "graph_coloring") { + ilu_reorder = bda::ILUReorder::GRAPH_COLORING; + } else { + OPM_THROW(std::logic_error, "Error invalid argument for --opencl-ilu-reorder, usage: '--opencl-ilu-reorder=[level_scheduling|graph_coloring]'"); + } + backend.reset(new bda::openclSolverBackend(linear_solver_verbosity, maxit, tolerance, platformID, deviceID, ilu_reorder)); #else OPM_THROW(std::logic_error, "Error openclSolver was chosen, but OpenCL was not found by CMake"); #endif @@ -217,7 +226,7 @@ void BdaBridge::get_result(BridgeVector template BdaBridge, std::allocator > >, \ Dune::BlockVector, std::allocator > >, \ n>::BdaBridge \ -(std::string gpu_mode_, int linear_solver_verbosity, int maxit, double tolerance, unsigned int platformID, unsigned int deviceID, std::string ilu_reorder_strategy); \ +(std::string gpu_mode_, int linear_solver_verbosity, int maxit, double tolerance, unsigned int platformID, unsigned int deviceID, std::string opencl_ilu_reorder); \ \ template void BdaBridge, std::allocator > >, \ Dune::BlockVector, std::allocator > >, \ diff --git a/opm/simulators/linalg/bda/BdaBridge.hpp b/opm/simulators/linalg/bda/BdaBridge.hpp index 3b368ecaf..18e623770 100644 --- a/opm/simulators/linalg/bda/BdaBridge.hpp +++ b/opm/simulators/linalg/bda/BdaBridge.hpp @@ -25,6 +25,7 @@ #include "dune/istl/bcrsmatrix.hh" #include +#include #include #if HAVE_CUDA @@ -39,6 +40,7 @@ namespace Opm { typedef Dune::InverseOperatorResult InverseOperatorResult; +using bda::ILUReorder; /// BdaBridge acts as interface between opm-simulators with the BdaSolvers template @@ -56,8 +58,8 @@ public: /// \param[in] tolerance required relative tolerance for BdaSolver /// \param[in] platformID the OpenCL platform ID to be used /// \param[in] deviceID the device ID to be used by the cusparse- and openclSolvers, too high values could cause runtime errors - /// \param[in] ilu_reorder_strategy select either level_scheduling or graph_coloring, see BILU0.hpp for explanation - BdaBridge(std::string gpu_mode, int linear_solver_verbosity, int maxit, double tolerance, unsigned int platformID, unsigned int deviceID, std::string ilu_reorder_strategy); + /// \param[in] opencl_ilu_reorder select either level_scheduling or graph_coloring, see BILU0.hpp for explanation + BdaBridge(std::string gpu_mode, int linear_solver_verbosity, int maxit, double tolerance, unsigned int platformID, unsigned int deviceID, std::string opencl_ilu_reorder); /// Solve linear system, A*x = b diff --git a/opm/simulators/linalg/bda/ILUReorder.hpp b/opm/simulators/linalg/bda/ILUReorder.hpp new file mode 100644 index 000000000..83039cf6d --- /dev/null +++ b/opm/simulators/linalg/bda/ILUReorder.hpp @@ -0,0 +1,35 @@ +/* + Copyright 2020 Equinor ASA + + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ + +#ifndef ILUREORDER_HEADER_INCLUDED +#define ILUREORDER_HEADER_INCLUDED + +namespace bda +{ + // Level Scheduling respects the dependencies in the original matrix, and behaves like Dune and cusparse + // Graph Coloring is more aggresive and is likely to increase the number of linearizations and linear iterations to converge significantly, but can still be faster on GPU because it results in more parallelism + + enum class ILUReorder { + LEVEL_SCHEDULING, + GRAPH_COLORING + }; + +} + +#endif diff --git a/opm/simulators/linalg/bda/openclSolverBackend.cpp b/opm/simulators/linalg/bda/openclSolverBackend.cpp index 8eb61c938..185b946af 100644 --- a/opm/simulators/linalg/bda/openclSolverBackend.cpp +++ b/opm/simulators/linalg/bda/openclSolverBackend.cpp @@ -44,8 +44,8 @@ using Opm::OpmLog; using Dune::Timer; template -openclSolverBackend::openclSolverBackend(int verbosity_, int maxit_, double tolerance_, unsigned int platformID_, unsigned int deviceID_, std::string ilu_reorder_strategy) : BdaSolver(verbosity_, maxit_, tolerance_, platformID_, deviceID_) { - prec = new Preconditioner(ilu_reorder_strategy, verbosity_); +openclSolverBackend::openclSolverBackend(int verbosity_, int maxit_, double tolerance_, unsigned int platformID_, unsigned int deviceID_, ILUReorder opencl_ilu_reorder) : BdaSolver(verbosity_, maxit_, tolerance_, platformID_, deviceID_) { + prec = new Preconditioner(opencl_ilu_reorder, verbosity_); wcontainer = new WContainer(); } @@ -730,7 +730,7 @@ SolverStatus openclSolverBackend::solve_system(int N_, int nnz_, int #define INSTANTIATE_BDA_FUNCTIONS(n) \ -template openclSolverBackend::openclSolverBackend(int, int, double, unsigned int, unsigned int, std::string); \ +template openclSolverBackend::openclSolverBackend(int, int, double, unsigned int, unsigned int, ILUReorder); \ INSTANTIATE_BDA_FUNCTIONS(1); INSTANTIATE_BDA_FUNCTIONS(2); diff --git a/opm/simulators/linalg/bda/openclSolverBackend.hpp b/opm/simulators/linalg/bda/openclSolverBackend.hpp index 7d6cf6938..6981c2801 100644 --- a/opm/simulators/linalg/bda/openclSolverBackend.hpp +++ b/opm/simulators/linalg/bda/openclSolverBackend.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -178,8 +179,8 @@ public: /// \param[in] tolerance required relative tolerance for openclSolver /// \param[in] platformID the OpenCL platform to be used /// \param[in] deviceID the device to be used - /// \param[in] ilu_reorder_strategy select either level_scheduling or graph_coloring, see BILU0.hpp for explanation - openclSolverBackend(int linear_solver_verbosity, int maxit, double tolerance, unsigned int platformID, unsigned int deviceID, std::string ilu_reorder_strategy); + /// \param[in] opencl_ilu_reorder select either level_scheduling or graph_coloring, see BILU0.hpp for explanation + openclSolverBackend(int linear_solver_verbosity, int maxit, double tolerance, unsigned int platformID, unsigned int deviceID, ILUReorder opencl_ilu_reorder); /// Destroy a openclSolver, and free memory ~openclSolverBackend();