Add tests for Hypre Preconditioner

CMakeLists.txt

@@ -726,12 +726,14 @@ if(USE_HYPRE)
   find_package(HYPRE)
   if(HYPRE_FOUND)
     set(HAVE_HYPRE 1)
+    target_link_libraries(opmsimulators PUBLIC HYPRE::HYPRE)
+    if (HYPRE_USING_CUDA)
+      set_tests_properties(HyprePreconditionerGPU PROPERTIES LABELS gpu_cuda)
+    elseif (HYPRE_USING_HIP)
+      set_tests_properties(HyprePreconditionerGPU PROPERTIES LABELS gpu_hip)
+    endif()
   else()
     message(WARNING "Hypre requested but not found. Continuing without Hypre support.")
     set(USE_HYPRE OFF)
   endif()
 endif()
-
-if(HYPRE_FOUND)
-  target_link_libraries(opmsimulators PUBLIC HYPRE::HYPRE)
-endif()

CMakeLists_files.cmake

@@ -492,6 +492,13 @@ if(HDF5_FOUND)
   list(APPEND TEST_SOURCE_FILES tests/test_HDF5Serializer.cpp)
 endif()
 
+if(HYPRE_FOUND)
+  list(APPEND TEST_SOURCE_FILES tests/test_HyprePreconditionerCPU.cpp)
+  if(HYPRE_USING_CUDA OR HYPRE_USING_HIP)
+    list(APPEND TEST_SOURCE_FILES tests/test_HyprePreconditionerGPU.cpp)
+  endif()
+endif()
+
 list (APPEND TEST_DATA_FILES
   tests/equil_base.DATA
   tests/equil_capillary.DATA

opm/simulators/linalg/HyprePreconditioner.hpp

@@ -1,14 +1,19 @@
 /*
   Copyright 2024 SINTEF AS
+  Copyright 2024 Equinor ASA
+
   This file is part of the Open Porous Media project (OPM).
+
   OPM is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.
+
   OPM is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
+
   You should have received a copy of the GNU General Public License
   along with OPM.  If not, see <http://www.gnu.org/licenses/>.
 */
@@ -19,6 +24,7 @@
 #include <opm/common/ErrorMacros.hpp>
 #include <opm/common/TimingMacros.hpp>
 #include <opm/simulators/linalg/PreconditionerWithUpdate.hpp>
+#include <opm/simulators/linalg/PropertyTree.hpp>
 
 #include <dune/common/fmatrix.hh>
 #include <dune/istl/bcrsmatrix.hh>
@@ -28,7 +34,6 @@
 #include <HYPRE_krylov.h>
 #include <_hypre_utilities.h>
 
-#include <memory>
 #include <vector>
 #include <numeric>
 
@@ -48,11 +53,17 @@ public:
     using field_type = typename X::field_type;
 
     // Constructor
-    HyprePreconditioner (const M& A, const Opm::PropertyTree& prm)
+    HyprePreconditioner (const M& A, const Opm::PropertyTree prm)
         : A_(A), prm_(prm)
     {
         OPM_TIMEBLOCK(prec_construct);
 
+        int size;
+        MPI_Comm_size(MPI_COMM_WORLD, &size);
+        if (size > 1) {
+            OPM_THROW(std::runtime_error, "HyprePreconditioner is currently only implemented for sequential runs");
+        }
+
         use_gpu_ = prm_.get<bool>("use_gpu", false);
 
         // Set memory location and execution policy
@@ -104,8 +115,8 @@ public:
         // Create Hypre vectors
         N_ = A_.N();
         nnz_ = A_.nonzeroes();
-        HYPRE_IJVectorCreate(MPI_COMM_WORLD, 0, N_-1, &x_hypre_);
+        HYPRE_IJVectorCreate(MPI_COMM_SELF, 0, N_-1, &x_hypre_);
-        HYPRE_IJVectorCreate(MPI_COMM_WORLD, 0, N_-1, &b_hypre_);
+        HYPRE_IJVectorCreate(MPI_COMM_SELF, 0, N_-1, &b_hypre_);
         HYPRE_IJVectorSetObjectType(x_hypre_, HYPRE_PARCSR);
         HYPRE_IJVectorSetObjectType(b_hypre_, HYPRE_PARCSR);
         HYPRE_IJVectorInitialize(x_hypre_);
@@ -122,7 +133,7 @@ public:
         }
 
         // Create Hypre matrix
-        HYPRE_IJMatrixCreate(MPI_COMM_WORLD, 0, N_-1, 0, N_-1, &A_hypre_);
+        HYPRE_IJMatrixCreate(MPI_COMM_SELF, 0, N_-1, 0, N_-1, &A_hypre_);
         HYPRE_IJMatrixSetObjectType(A_hypre_, HYPRE_PARCSR);
         HYPRE_IJMatrixInitialize(A_hypre_);
 
@@ -164,9 +175,7 @@ public:
         HYPRE_BoomerAMGSetup(solver_, parcsr_A_, par_b_, par_x_);
     }
 
-    void pre(X& x, Y& b) override {
+    void pre(X& /*x*/, Y& /*b*/) override {
-        DUNE_UNUSED_PARAMETER(x);
-        DUNE_UNUSED_PARAMETER(b);
     }
 
     void apply(X& v, const Y& d) override {
@@ -182,8 +191,7 @@ public:
         copyVectorFromHypre(v);
     }
 
-    void post(X& x) override {
+    void post(X& /*x*/) override {
-        DUNE_UNUSED_PARAMETER(x);
     }
 
     Dune::SolverCategory::Category category() const override {

tests/HyprePreconditionerTestHelper.hpp

@@ -0,0 +1,123 @@
+/*
+  Copyright 2024 SINTEF AS
+  Copyright 2024 Equinor ASA
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef TEST_HYPREPRECONDITIONER_HELPER_HPP
+#define TEST_HYPREPRECONDITIONER_HELPER_HPP
+
+#include <boost/test/unit_test.hpp>
+
+#include <dune/common/fmatrix.hh>
+#include <dune/istl/bcrsmatrix.hh>
+#include <dune/istl/bvector.hh>
+#include <dune/istl/operators.hh>
+#include <dune/istl/solvers.hh>
+
+#include <opm/simulators/linalg/HyprePreconditioner.hpp>
+#include <opm/simulators/linalg/PropertyTree.hpp>
+
+
+template<class Matrix>
+void setupLaplace2d(int N, Matrix& mat)
+{
+    const int nonZeroes = N*N * 5; // max 5 entries per row (diagonal + 4 neighbors)
+    mat.setBuildMode(Matrix::row_wise);
+    mat.setSize(N*N, N*N, nonZeroes);
+
+    // Set up sparsity pattern
+    for (auto row = mat.createbegin(); row != mat.createend(); ++row) {
+        const int i = row.index();
+        int x = i % N;
+        int y = i / N;
+
+        row.insert(i);  // diagonal
+        if (x > 0)      // left neighbor
+            row.insert(i-1);
+        if (x < N-1)    // right neighbor
+            row.insert(i+1);
+        if (y > 0)      // upper neighbor
+            row.insert(i-N);
+        if (y < N-1)    // lower neighbor
+            row.insert(i+N);
+    }
+
+    // Fill the matrix with values
+    for (auto row = mat.begin(); row != mat.end(); ++row) {
+        const int i = row.index();
+        int x = i % N;
+        int y = i / N;
+
+        // Set diagonal
+        (*row)[i] = 4.0;
+
+        // Set off-diagonal entries
+        if (x > 0)         // left neighbor
+            (*row)[i-1] = -1.0;
+        if (x < N-1)       // right neighbor
+            (*row)[i+1] = -1.0;
+        if (y > 0)         // upper neighbor
+            (*row)[i-N] = -1.0;
+        if (y < N-1)       // lower neighbor
+            (*row)[i+N] = -1.0;
+    }
+}
+
+inline void testHyprePreconditioner(bool use_gpu)
+{
+    constexpr int N = 100; // 100x100 grid
+    using Matrix = Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>>;
+    using Vector = Dune::BlockVector<Dune::FieldVector<double, 1>>;
+
+    // Create matrix
+    Matrix matrix;
+    setupLaplace2d(N, matrix);
+
+    // Create vectors
+    Vector x(N*N), b(N*N);
+    x = 100.0;  // Initial guess
+    b = 0.0;    // RHS
+
+    // Create operator
+    using Operator = Dune::MatrixAdapter<Matrix, Vector, Vector>;
+    Operator op(matrix);
+
+    // Set up HYPRE parameters
+    Opm::PropertyTree prm;
+    prm.put("use_gpu", use_gpu ? 1 : 0);
+
+    // Create preconditioner
+    auto prec = std::make_shared<Hypre::HyprePreconditioner<Matrix, Vector, Vector>>(matrix, prm);
+
+    // Create solver
+    double reduction = 1e-8;
+    int maxit = 300;
+    int verbosity = 0;
+    Dune::LoopSolver<Vector> solver(op, *prec, reduction, maxit, verbosity);
+
+    // Solve
+    Dune::InverseOperatorResult res;
+    solver.apply(x, b, res);
+
+    // Check convergence
+    BOOST_CHECK(res.converged);
+    BOOST_CHECK_LT(res.reduction, 1e-8);
+}
+
+
+#endif // TEST_HYPREPRECONDITIONER_HELPER_HPP

tests/test_HyprePreconditionerCPU.cpp

@@ -0,0 +1,68 @@
+/*
+  Copyright 2024 SINTEF AS
+  Copyright 2024 Equinor ASA
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+
+#define BOOST_TEST_MODULE TestHyprePreconditionerCPU
+#define BOOST_TEST_NO_MAIN
+
+#include <dune/common/parallel/mpihelper.hh>
+#include "MpiFixture.hpp"
+#include "HyprePreconditionerTestHelper.hpp"
+
+#include <boost/test/unit_test.hpp>
+
+#include <dune/common/fmatrix.hh>
+#include <dune/istl/bcrsmatrix.hh>
+#include <dune/istl/bvector.hh>
+#include <dune/istl/operators.hh>
+#include <dune/istl/solvers.hh>
+
+#include <opm/simulators/linalg/HyprePreconditioner.hpp>
+#include <opm/simulators/linalg/PropertyTree.hpp>
+
+BOOST_GLOBAL_FIXTURE(MPIFixture);
+
+BOOST_AUTO_TEST_CASE(TestHyprePreconditionerCPU)
+{
+    testHyprePreconditioner(false);
+}
+
+bool init_unit_test_func()
+{
+    return true;
+}
+
+int main(int argc, char** argv)
+{
+    Dune::MPIHelper::instance(argc, argv);
+
+#if HYPRE_RELEASE_NUMBER >= 22900
+    HYPRE_Initialize();
+#else
+    HYPRE_Init();
+#endif
+
+    int result = boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv);
+
+    HYPRE_Finalize();
+
+    return result;
+}

tests/test_HyprePreconditionerGPU.cpp

@@ -0,0 +1,60 @@
+/*
+  Copyright 2024 SINTEF AS
+  Copyright 2024 Equinor ASA
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+
+#define BOOST_TEST_MODULE TestHyprePreconditionerGPU
+#define BOOST_TEST_NO_MAIN
+#include <boost/test/unit_test.hpp>
+
+#include <opm/simulators/linalg/HyprePreconditioner.hpp>
+#include <dune/common/parallel/mpihelper.hh>
+
+#include "MpiFixture.hpp"
+#include "HyprePreconditionerTestHelper.hpp"
+
+BOOST_GLOBAL_FIXTURE(MPIFixture);
+
+BOOST_AUTO_TEST_CASE(TestHyprePreconditionerGPU)
+{
+    testHyprePreconditioner(true);
+}
+
+bool init_unit_test_func()
+{
+    return true;
+}
+
+int main(int argc, char** argv)
+{
+    Dune::MPIHelper::instance(argc, argv);
+
+#if HYPRE_RELEASE_NUMBER >= 22900
+    HYPRE_Initialize();
+#else
+    HYPRE_Init();
+#endif
+
+    int result = boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv);
+
+    HYPRE_Finalize();
+
+    return result;
+}