From 0e63cda5182cfdb57012c226253b23176ef11da3 Mon Sep 17 00:00:00 2001 From: Tong Dong Qiu Date: Tue, 1 Jun 2021 13:12:52 +0200 Subject: [PATCH 1/5] Instantiate whole class instead of each function --- opm/simulators/linalg/bda/BdaBridge.cpp | 27 +++++-------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/opm/simulators/linalg/bda/BdaBridge.cpp b/opm/simulators/linalg/bda/BdaBridge.cpp index 48717fd4e..62dc6c6ba 100644 --- a/opm/simulators/linalg/bda/BdaBridge.cpp +++ b/opm/simulators/linalg/bda/BdaBridge.cpp @@ -276,28 +276,11 @@ void BdaBridge::initWellContributions([[ } } -#define INSTANTIATE_BDA_FUNCTIONS(n) \ -template BdaBridge, std::allocator > >, \ -Dune::BlockVector, std::allocator > >, \ -n>::BdaBridge \ -(std::string accelerator_mode_, std::string fpga_bitstream, int linear_solver_verbosity, int maxit, double tolerance, \ -unsigned int platformID, unsigned int deviceID, std::string opencl_ilu_reorder); \ - \ -template void BdaBridge, std::allocator > >, \ -Dune::BlockVector, std::allocator > >, \ -n>::solve_system \ -(Dune::BCRSMatrix, std::allocator > >*, \ - Dune::BlockVector, std::allocator > >&, \ - WellContributions&, InverseOperatorResult&); \ - \ -template void BdaBridge, std::allocator > >, \ -Dune::BlockVector, std::allocator > >, \ -n>::get_result \ -(Dune::BlockVector, std::allocator > >&); \ - \ -template void BdaBridge, std::allocator > >, \ -Dune::BlockVector, std::allocator > >, \ -n>::initWellContributions(WellContributions&) + +#define INSTANTIATE_BDA_FUNCTIONS(n) \ +template class BdaBridge, std::allocator > >, \ +Dune::BlockVector, std::allocator > >, \ +n>; INSTANTIATE_BDA_FUNCTIONS(1); From 74b46e4d8d8bf83145da821598f1be2d7c9122d3 Mon Sep 17 00:00:00 2001 From: Tong Dong Qiu Date: Tue, 1 Jun 2021 13:50:27 +0200 Subject: [PATCH 2/5] Added tests for cusparseSolver and openclSolver --- CMakeLists_files.cmake | 6 ++ opm/simulators/linalg/bda/BdaBridge.cpp | 12 ++- tests/test_cusparseSolver.cpp | 127 ++++++++++++++++++++++++ tests/test_openclSolver.cpp | 126 +++++++++++++++++++++++ 4 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 tests/test_cusparseSolver.cpp create mode 100644 tests/test_openclSolver.cpp diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index a21c44383..9327f0d2c 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -140,6 +140,12 @@ if(MPI_FOUND) list(APPEND TEST_SOURCE_FILES tests/test_parallelistlinformation.cpp tests/test_ParallelRestart.cpp) endif() +if(CUDA_FOUND) + list(APPEND TEST_SOURCE_FILES tests/test_cusparseSolver.cpp) +endif() +if(OPENCL_FOUND) + list(APPEND TEST_SOURCE_FILES tests/test_openclSolver.cpp) +endif() list (APPEND TEST_DATA_FILES tests/SUMMARY_DECK_NON_CONSTANT_POROSITY.DATA diff --git a/opm/simulators/linalg/bda/BdaBridge.cpp b/opm/simulators/linalg/bda/BdaBridge.cpp index 62dc6c6ba..a20b36524 100644 --- a/opm/simulators/linalg/bda/BdaBridge.cpp +++ b/opm/simulators/linalg/bda/BdaBridge.cpp @@ -276,10 +276,14 @@ void BdaBridge::initWellContributions([[ } } - -#define INSTANTIATE_BDA_FUNCTIONS(n) \ -template class BdaBridge, std::allocator > >, \ -Dune::BlockVector, std::allocator > >, \ +// the tests use Dune::FieldMatrix, Flow uses Opm::MatrixBlock +#define INSTANTIATE_BDA_FUNCTIONS(n) \ +template class BdaBridge, std::allocator > >, \ +Dune::BlockVector, std::allocator > >, \ +n>; \ + \ +template class BdaBridge, std::allocator > >, \ +Dune::BlockVector, std::allocator > >, \ n>; diff --git a/tests/test_cusparseSolver.cpp b/tests/test_cusparseSolver.cpp new file mode 100644 index 000000000..5676cb6f5 --- /dev/null +++ b/tests/test_cusparseSolver.cpp @@ -0,0 +1,127 @@ +/* + Copyright 2021 Equinor + + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ + +#include + +#define BOOST_TEST_MODULE OPM_test_cusparseSolver +#include +#include + +#include + +#if DUNE_VERSION_NEWER(DUNE_ISTL, 2, 6) && \ + BOOST_VERSION / 100 % 1000 > 48 + +#include + +#include +#include +#include +#include +#include + +#include +#include + + +template +Dune::BlockVector> +testCusparseSolver(const boost::property_tree::ptree& prm, const std::string& matrix_filename, const std::string& rhs_filename) +{ + using Matrix = Dune::BCRSMatrix>; + using Vector = Dune::BlockVector>; + Matrix matrix; + { + std::ifstream mfile(matrix_filename); + if (!mfile) { + throw std::runtime_error("Could not read matrix file"); + } + readMatrixMarket(matrix, mfile); + } + Vector rhs; + { + std::ifstream rhsfile(rhs_filename); + if (!rhsfile) { + throw std::runtime_error("Could not read rhs file"); + } + readMatrixMarket(rhs, rhsfile); + } + + const int linear_solver_verbosity = prm.get("verbosity"); + const int maxit = prm.get("maxiter"); + const double tolerance = prm.get("tol"); + const std::string opencl_ilu_reorder("none"); // unused + const int platformID = 0; // unused + const int deviceID = 0; + const std::string gpu_mode("cusparse"); + const std::string fpga_bitstream("empty"); // unused + + Vector x(rhs.size()); + Opm::WellContributions wellContribs("cusparse"); + Opm::BdaBridge bridge(gpu_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder); + Dune::InverseOperatorResult result; + + bridge.solve_system(&matrix, rhs, wellContribs, result); + bridge.get_result(x); + + return x; +} + +namespace pt = boost::property_tree; + +void test3(const pt::ptree& prm) +{ + const int bz = 3; + auto sol = testCusparseSolver(prm, "matr33.txt", "rhs3.txt"); + Dune::BlockVector> expected {{-0.0131626, -3.5826e-6, 1.138362e-9}, + {-1.25425e-3, -1.4167e-4, -0.0029366}, + {-4.54355e-4, 1.28682e-5, 4.7644e-6}}; + BOOST_REQUIRE_EQUAL(sol.size(), expected.size()); + for (size_t i = 0; i < sol.size(); ++i) { + for (int row = 0; row < bz; ++row) { + BOOST_CHECK_CLOSE(sol[i][row], expected[i][row], 1e-3); + } + } +} + + +BOOST_AUTO_TEST_CASE(TestDefaultPreconditionerFactory) +{ + pt::ptree prm; + + // Read parameters. + { + std::ifstream file("options_flexiblesolver.json"); + pt::read_json(file, prm); + } + + // Test with 3x3 block solvers. + test3(prm); +} + + +#else + +// Do nothing if we do not have at least Dune 2.6. +BOOST_AUTO_TEST_CASE(DummyTest) +{ + BOOST_REQUIRE(true); +} + +#endif diff --git a/tests/test_openclSolver.cpp b/tests/test_openclSolver.cpp new file mode 100644 index 000000000..8fed2bdcb --- /dev/null +++ b/tests/test_openclSolver.cpp @@ -0,0 +1,126 @@ +/* + Copyright 2021 Equinor + + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ + +#include + +#define BOOST_TEST_MODULE OPM_test_openclSolver +#include +#include + +#include + +#if DUNE_VERSION_NEWER(DUNE_ISTL, 2, 6) && \ + BOOST_VERSION / 100 % 1000 > 48 + +#include + +#include +#include +#include +#include + +#include +#include + + +template +Dune::BlockVector> +testOpenclSolver(const boost::property_tree::ptree& prm, const std::string& matrix_filename, const std::string& rhs_filename) +{ + using Matrix = Dune::BCRSMatrix>; + using Vector = Dune::BlockVector>; + Matrix matrix; + { + std::ifstream mfile(matrix_filename); + if (!mfile) { + throw std::runtime_error("Could not read matrix file"); + } + readMatrixMarket(matrix, mfile); + } + Vector rhs; + { + std::ifstream rhsfile(rhs_filename); + if (!rhsfile) { + throw std::runtime_error("Could not read rhs file"); + } + readMatrixMarket(rhs, rhsfile); + } + + const int linear_solver_verbosity = prm.get("verbosity"); + const int maxit = prm.get("maxiter"); + const double tolerance = prm.get("tol"); + const std::string opencl_ilu_reorder("none"); + const int platformID = 0; + const int deviceID = 0; + const std::string gpu_mode("opencl"); + const std::string fpga_bitstream("empty"); // unused + + Vector x(rhs.size()); + Opm::WellContributions wellContribs("opencl"); + Opm::BdaBridge bridge(gpu_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder); + Dune::InverseOperatorResult result; + + bridge.solve_system(&matrix, rhs, wellContribs, result); + bridge.get_result(x); + + return x; +} + +namespace pt = boost::property_tree; + +void test3(const pt::ptree& prm) +{ + const int bz = 3; + auto sol = testOpenclSolver(prm, "matr33.txt", "rhs3.txt"); + Dune::BlockVector> expected {{-1.30307e-2, -3.58263e-6, 1.13836e-9}, + {-1.25425e-3, -1.4167e-4, -3.2213e-3}, + {-4.5436e-4, 1.28682e-5, 4.7644e-6}}; + BOOST_REQUIRE_EQUAL(sol.size(), expected.size()); + for (size_t i = 0; i < sol.size(); ++i) { + for (int row = 0; row < bz; ++row) { + BOOST_CHECK_CLOSE(sol[i][row], expected[i][row], 1e-3); + } + } +} + + +BOOST_AUTO_TEST_CASE(TestDefaultPreconditionerFactory) +{ + pt::ptree prm; + + // Read parameters. + { + std::ifstream file("options_flexiblesolver.json"); + pt::read_json(file, prm); + } + + // Test with 3x3 block solvers. + test3(prm); +} + + +#else + +// Do nothing if we do not have at least Dune 2.6. +BOOST_AUTO_TEST_CASE(DummyTest) +{ + BOOST_REQUIRE(true); +} + +#endif From 393ab24154534811e9c917f906e59a6493dba901 Mon Sep 17 00:00:00 2001 From: Tong Dong Qiu Date: Tue, 1 Jun 2021 13:50:47 +0200 Subject: [PATCH 3/5] Updated copyright notice --- tests/test_cusparseSolver.cpp | 1 + tests/test_openclSolver.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/test_cusparseSolver.cpp b/tests/test_cusparseSolver.cpp index 5676cb6f5..5bace763d 100644 --- a/tests/test_cusparseSolver.cpp +++ b/tests/test_cusparseSolver.cpp @@ -1,4 +1,5 @@ /* + Copyright 2019 SINTEF Digital, Mathematics and Cybernetics. Copyright 2021 Equinor This file is part of the Open Porous Media project (OPM). diff --git a/tests/test_openclSolver.cpp b/tests/test_openclSolver.cpp index 8fed2bdcb..4eceeb0c9 100644 --- a/tests/test_openclSolver.cpp +++ b/tests/test_openclSolver.cpp @@ -1,4 +1,5 @@ /* + Copyright 2019 SINTEF Digital, Mathematics and Cybernetics. Copyright 2021 Equinor This file is part of the Open Porous Media project (OPM). From 31eeb330396f3eae36cff28d2d56009db6f81864 Mon Sep 17 00:00:00 2001 From: Tong Dong Qiu Date: Tue, 8 Jun 2021 16:07:38 +0200 Subject: [PATCH 4/5] Added error checking to tests --- tests/test_cusparseSolver.cpp | 12 ++++++++---- tests/test_openclSolver.cpp | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/test_cusparseSolver.cpp b/tests/test_cusparseSolver.cpp index 5bace763d..f1ed325e9 100644 --- a/tests/test_cusparseSolver.cpp +++ b/tests/test_cusparseSolver.cpp @@ -72,14 +72,18 @@ testCusparseSolver(const boost::property_tree::ptree& prm, const std::string& ma const int deviceID = 0; const std::string gpu_mode("cusparse"); const std::string fpga_bitstream("empty"); // unused + Dune::InverseOperatorResult result; Vector x(rhs.size()); Opm::WellContributions wellContribs("cusparse"); - Opm::BdaBridge bridge(gpu_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder); - Dune::InverseOperatorResult result; + try { + Opm::BdaBridge bridge(gpu_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder); - bridge.solve_system(&matrix, rhs, wellContribs, result); - bridge.get_result(x); + bridge.solve_system(&matrix, rhs, wellContribs, result); + bridge.get_result(x); + } catch (const std::logic_error& error) { + BOOST_WARN_MESSAGE(true, error.what()); + } return x; } diff --git a/tests/test_openclSolver.cpp b/tests/test_openclSolver.cpp index 4eceeb0c9..fd2c7fdb4 100644 --- a/tests/test_openclSolver.cpp +++ b/tests/test_openclSolver.cpp @@ -71,14 +71,18 @@ testOpenclSolver(const boost::property_tree::ptree& prm, const std::string& matr const int deviceID = 0; const std::string gpu_mode("opencl"); const std::string fpga_bitstream("empty"); // unused + Dune::InverseOperatorResult result; Vector x(rhs.size()); Opm::WellContributions wellContribs("opencl"); - Opm::BdaBridge bridge(gpu_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder); - Dune::InverseOperatorResult result; + try { + Opm::BdaBridge bridge(gpu_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder); - bridge.solve_system(&matrix, rhs, wellContribs, result); - bridge.get_result(x); + bridge.solve_system(&matrix, rhs, wellContribs, result); + bridge.get_result(x); + } catch (const std::logic_error& error) { + BOOST_WARN_MESSAGE(true, error.what()); + } return x; } From f075c236c11bca2a32befe368c0deefc4b1b51c6 Mon Sep 17 00:00:00 2001 From: Tong Dong Qiu Date: Wed, 16 Jun 2021 15:54:06 +0200 Subject: [PATCH 5/5] New error handling from Markus --- tests/test_openclSolver.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/test_openclSolver.cpp b/tests/test_openclSolver.cpp index fd2c7fdb4..94bb0b88d 100644 --- a/tests/test_openclSolver.cpp +++ b/tests/test_openclSolver.cpp @@ -39,6 +39,11 @@ #include #include +class PlatformInitException : public std::logic_error +{ +public: + PlatformInitException(std::string msg) : logic_error(msg){}; +}; template Dune::BlockVector> @@ -75,14 +80,15 @@ testOpenclSolver(const boost::property_tree::ptree& prm, const std::string& matr Vector x(rhs.size()); Opm::WellContributions wellContribs("opencl"); + std::unique_ptr > bridge; try { - Opm::BdaBridge bridge(gpu_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder); - - bridge.solve_system(&matrix, rhs, wellContribs, result); - bridge.get_result(x); + bridge = std::make_unique >(gpu_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder); } catch (const std::logic_error& error) { BOOST_WARN_MESSAGE(true, error.what()); + throw PlatformInitException(error.what()); } + bridge->solve_system(&matrix, rhs, wellContribs, result); + bridge->get_result(x); return x; } @@ -115,8 +121,12 @@ BOOST_AUTO_TEST_CASE(TestDefaultPreconditionerFactory) pt::read_json(file, prm); } - // Test with 3x3 block solvers. - test3(prm); + try { + // Test with 3x3 block solvers. + test3(prm); + } catch(const PlatformInitException& ) { + BOOST_WARN_MESSAGE(true, "Problem with initializing Platform. skipping test"); + } }