From aa362ae2391ed1311f389d3bc23c900c7a2398cb Mon Sep 17 00:00:00 2001 From: Tong Dong Qiu Date: Tue, 29 Mar 2022 14:38:45 +0200 Subject: [PATCH] Fix GPU tests, use Dune as reference instead of hardcoded --- tests/test_cusparseSolver.cpp | 70 +++++++++++++++++++++++++---------- tests/test_openclSolver.cpp | 64 ++++++++++++++++++++++++-------- 2 files changed, 99 insertions(+), 35 deletions(-) diff --git a/tests/test_cusparseSolver.cpp b/tests/test_cusparseSolver.cpp index b75cc99f8..6c533134e 100644 --- a/tests/test_cusparseSolver.cpp +++ b/tests/test_cusparseSolver.cpp @@ -36,7 +36,8 @@ #include #include #include -#include +#include +#include #include #include @@ -48,12 +49,13 @@ public: }; template -Dune::BlockVector> -testCusparseSolver(const boost::property_tree::ptree& prm, const std::string& matrix_filename, const std::string& rhs_filename) +using Matrix = Dune::BCRSMatrix>; +template +using Vector = Dune::BlockVector>; + +template +void readLinearSystem(const std::string& matrix_filename, const std::string& rhs_filename, Matrix& matrix, Vector& rhs) { - using Matrix = Dune::BCRSMatrix>; - using Vector = Dune::BlockVector>; - Matrix matrix; { std::ifstream mfile(matrix_filename); if (!mfile) { @@ -61,7 +63,6 @@ testCusparseSolver(const boost::property_tree::ptree& prm, const std::string& ma } readMatrixMarket(matrix, mfile); } - Vector rhs; { std::ifstream rhsfile(rhs_filename); if (!rhsfile) { @@ -69,6 +70,32 @@ testCusparseSolver(const boost::property_tree::ptree& prm, const std::string& ma } readMatrixMarket(rhs, rhsfile); } +} + +template +Dune::BlockVector> +getDuneSolution(Matrix& matrix, Vector& rhs) +{ + Dune::InverseOperatorResult result; + + Vector x(rhs.size()); + + typedef Dune::MatrixAdapter,Vector,Vector > Operator; + Operator fop(matrix); + double relaxation = 0.9; + Dune::SeqILU,Vector,Vector > prec(matrix, relaxation); + double reduction = 1e-2; + int maxit = 10; + int verbosity = 0; + Dune::BiCGSTABSolver > solver(fop, prec, reduction, maxit, verbosity); + solver.apply(x, rhs, result); + return x; +} + +template +Dune::BlockVector> +testCusparseSolver(const boost::property_tree::ptree& prm, Matrix& matrix, Vector& rhs) +{ const int linear_solver_verbosity = prm.get("verbosity"); const int maxit = prm.get("maxiter"); @@ -81,13 +108,15 @@ testCusparseSolver(const boost::property_tree::ptree& prm, const std::string& ma const std::string linsolver("ilu0"); Dune::InverseOperatorResult result; - Vector x(rhs.size()); - auto wellContribs = Opm::WellContributions::create("cusparse", false); - std::unique_ptr > bridge; - try { - bridge = std::make_unique >(accelerator_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder, linsolver); + Vector x(rhs.size()); - bridge->solve_system(&matrix, rhs, *wellContribs, result); + auto wellContribs = Opm::WellContributions::create("cusparse", false); + std::unique_ptr, Vector, bz> > bridge; + try { + bridge = std::make_unique, Vector, bz> >(accelerator_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder, linsolver); + auto mat2 = matrix; // deep copy to make sure nnz values are in contiguous memory + // matrix created by readMatrixMarket() did not have contiguous memory + bridge->solve_system(&mat2, rhs, *wellContribs, result); bridge->get_result(x); return x; @@ -105,14 +134,17 @@ namespace pt = boost::property_tree; void test3(const pt::ptree& prm) { const int bz = 3; - auto sol = testCusparseSolver(prm, "matr33.txt", "rhs3.txt"); - Dune::BlockVector> expected {{-0.0131626, -3.5826e-6, 1.138362e-9}, - {-1.25425e-3, -1.4167e-4, -0.0029366}, - {-4.54355e-4, 1.28682e-5, 4.7644e-6}}; - BOOST_REQUIRE_EQUAL(sol.size(), expected.size()); + Matrix matrix; + Vector rhs; + readLinearSystem("matr33.txt", "rhs3.txt", matrix, rhs); + Vector rhs2 = rhs; // deep copy, getDuneSolution() changes values in rhs vector + auto duneSolution = getDuneSolution(matrix, rhs); + auto sol = testCusparseSolver(prm, matrix, rhs2); + + BOOST_REQUIRE_EQUAL(sol.size(), duneSolution.size()); for (size_t i = 0; i < sol.size(); ++i) { for (int row = 0; row < bz; ++row) { - BOOST_CHECK_CLOSE(sol[i][row], expected[i][row], 1e-3); + BOOST_CHECK_CLOSE(sol[i][row], duneSolution[i][row], 1e-3); } } } diff --git a/tests/test_openclSolver.cpp b/tests/test_openclSolver.cpp index 8ca40e024..94709ce3c 100644 --- a/tests/test_openclSolver.cpp +++ b/tests/test_openclSolver.cpp @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include @@ -47,12 +49,13 @@ public: }; template -Dune::BlockVector> -testOpenclSolver(const boost::property_tree::ptree& prm, const std::string& matrix_filename, const std::string& rhs_filename) +using Matrix = Dune::BCRSMatrix>; +template +using Vector = Dune::BlockVector>; + +template +void readLinearSystem(const std::string& matrix_filename, const std::string& rhs_filename, Matrix& matrix, Vector& rhs) { - using Matrix = Dune::BCRSMatrix>; - using Vector = Dune::BlockVector>; - Matrix matrix; { std::ifstream mfile(matrix_filename); if (!mfile) { @@ -60,7 +63,6 @@ testOpenclSolver(const boost::property_tree::ptree& prm, const std::string& matr } readMatrixMarket(matrix, mfile); } - Vector rhs; { std::ifstream rhsfile(rhs_filename); if (!rhsfile) { @@ -68,7 +70,32 @@ testOpenclSolver(const boost::property_tree::ptree& prm, const std::string& matr } readMatrixMarket(rhs, rhsfile); } +} +template +Dune::BlockVector> +getDuneSolution(Matrix& matrix, Vector& rhs) +{ + Dune::InverseOperatorResult result; + + Vector x(rhs.size()); + + typedef Dune::MatrixAdapter,Vector,Vector > Operator; + Operator fop(matrix); + double relaxation = 0.9; + Dune::SeqILU,Vector,Vector > prec(matrix, relaxation); + double reduction = 1e-2; + int maxit = 10; + int verbosity = 0; + Dune::BiCGSTABSolver > solver(fop, prec, reduction, maxit, verbosity); + solver.apply(x, rhs, result); + return x; +} + +template +Dune::BlockVector> +testOpenclSolver(const boost::property_tree::ptree& prm, Matrix& matrix, Vector& rhs) +{ const int linear_solver_verbosity = prm.get("verbosity"); const int maxit = prm.get("maxiter"); const double tolerance = prm.get("tol"); @@ -80,16 +107,18 @@ testOpenclSolver(const boost::property_tree::ptree& prm, const std::string& matr const std::string linsolver("ilu0"); Dune::InverseOperatorResult result; - Vector x(rhs.size()); + Vector x(rhs.size()); auto wellContribs = Opm::WellContributions::create("opencl", false); - std::unique_ptr > bridge; + std::unique_ptr, Vector, bz> > bridge; try { - bridge = std::make_unique >(accelerator_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder, linsolver); + bridge = std::make_unique, Vector, bz> >(accelerator_mode, fpga_bitstream, linear_solver_verbosity, maxit, tolerance, platformID, deviceID, opencl_ilu_reorder, linsolver); } catch (const std::logic_error& error) { BOOST_WARN_MESSAGE(true, error.what()); throw PlatformInitException(error.what()); } - bridge->solve_system(&matrix, rhs, *wellContribs, result); + auto mat2 = matrix; // deep copy to make sure nnz values are in contiguous memory + // matrix created by readMatrixMarket() did not have contiguous memory + bridge->solve_system(&mat2, rhs, *wellContribs, result); bridge->get_result(x); return x; @@ -100,14 +129,17 @@ namespace pt = boost::property_tree; void test3(const pt::ptree& prm) { const int bz = 3; - auto sol = testOpenclSolver(prm, "matr33.txt", "rhs3.txt"); - Dune::BlockVector> expected {{-0.0131626, -3.58263e-6, 1.13836e-9}, - {-1.25425e-3, -1.4167e-4, -0.0029366}, - {-4.5436e-4, 1.28682e-5, 4.7644e-6}}; - BOOST_REQUIRE_EQUAL(sol.size(), expected.size()); + Matrix matrix; + Vector rhs; + readLinearSystem("matr33.txt", "rhs3.txt", matrix, rhs); + Vector rhs2 = rhs; // deep copy, getDuneSolution() changes values in rhs vector + auto duneSolution = getDuneSolution(matrix, rhs); + auto sol = testOpenclSolver(prm, matrix, rhs2); + + BOOST_REQUIRE_EQUAL(sol.size(), duneSolution.size()); for (size_t i = 0; i < sol.size(); ++i) { for (int row = 0; row < bz; ++row) { - BOOST_CHECK_CLOSE(sol[i][row], expected[i][row], 1e-3); + BOOST_CHECK_CLOSE(sol[i][row], duneSolution[i][row], 1e-3); } } }