From ed8f37ec82ed0bf5f819420af9de63dfd5835ec1 Mon Sep 17 00:00:00 2001
From: Tong Dong Qiu <tongdong.qiu@bigdataccelerate.com>
Date: Thu, 2 Mar 2023 13:37:58 +0100
Subject: [PATCH] Initialize HIP/rocsparse in constructor

---
 .../linalg/bda/rocsparseSolverBackend.cpp     | 34 +++++++++++--------
 tests/test_rocsparseSolver.cpp                |  2 +-
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/opm/simulators/linalg/bda/rocsparseSolverBackend.cpp b/opm/simulators/linalg/bda/rocsparseSolverBackend.cpp
index e82c1612f..45284061a 100644
--- a/opm/simulators/linalg/bda/rocsparseSolverBackend.cpp
+++ b/opm/simulators/linalg/bda/rocsparseSolverBackend.cpp
@@ -72,11 +72,29 @@ using Dune::Timer;
 
 template <unsigned int block_size>
 rocsparseSolverBackend<block_size>::rocsparseSolverBackend(int verbosity_, int maxit_, double tolerance_, unsigned int platformID_, unsigned int deviceID_) : BdaSolver<block_size>(verbosity_, maxit_, tolerance_, platformID_, deviceID_) {
+    hipDevice_t device;
+    if(hipDeviceGet(&device, deviceID) != hipSuccess)
+    {
+        OPM_THROW(std::logic_error, "HIP Error: could not get device");
+    }
 
+    ROCSPARSE_CHECK(rocsparse_create_handle(&handle));
+    ROCBLAS_CHECK(rocblas_create_handle(&blas_handle));
+
+    ROCSPARSE_CHECK(rocsparse_get_version(handle, &ver));
+    ROCSPARSE_CHECK(rocsparse_get_git_rev(handle, rev));
+
+    std::ostringstream out;
+    out << "rocSPARSE version: " << ver / 100000 << "." << ver / 100 % 1000 << "."
+        << ver % 100 << "-" << rev << "\n";
+    OpmLog::info(out.str());
+
+    HIP_CHECK(hipStreamCreate(&stream));
+    ROCSPARSE_CHECK(rocsparse_set_stream(handle, stream));
+    ROCBLAS_CHECK(rocblas_set_stream(blas_handle, stream));
 }
 
 
-
 template <unsigned int block_size>
 rocsparseSolverBackend<block_size>::~rocsparseSolverBackend() {
     HIP_CHECK(hipStreamSynchronize(stream));
@@ -290,20 +308,6 @@ void rocsparseSolverBackend<block_size>::initialize(std::shared_ptr<BlockedMatri
     HIP_CHECK(hipMalloc((void**)&d_t, sizeof(double) * N));
     HIP_CHECK(hipMalloc((void**)&d_v, sizeof(double) * N));
 
-    ROCSPARSE_CHECK(rocsparse_create_handle(&handle));
-    ROCBLAS_CHECK(rocblas_create_handle(&blas_handle));
-
-    ROCSPARSE_CHECK(rocsparse_get_version(handle, &ver));
-    ROCSPARSE_CHECK(rocsparse_get_git_rev(handle, rev));
-
-    out << "rocSPARSE version: " << ver / 100000 << "." << ver / 100 % 1000 << "."
-        << ver % 100 << "-" << rev << "\n";
-    OpmLog::info(out.str());
-
-    HIP_CHECK(hipStreamCreate(&stream));
-    ROCSPARSE_CHECK(rocsparse_set_stream(handle, stream));
-    ROCBLAS_CHECK(rocblas_set_stream(blas_handle, stream));
-
     HIP_CHECK(hipMalloc((void**)&d_Arows, sizeof(rocsparse_int) * (Nb + 1)));
     HIP_CHECK(hipMalloc((void**)&d_Acols, sizeof(rocsparse_int) * nnzb));
     HIP_CHECK(hipMalloc((void**)&d_Avals, sizeof(double) * nnz));
diff --git a/tests/test_rocsparseSolver.cpp b/tests/test_rocsparseSolver.cpp
index 7bc5fad96..9b9302c80 100644
--- a/tests/test_rocsparseSolver.cpp
+++ b/tests/test_rocsparseSolver.cpp
@@ -114,7 +114,7 @@ createBridge(const boost::property_tree::ptree& prm, std::unique_ptr<Opm::BdaBri
                                                                                linsolver);
     } catch (const std::logic_error& error) {
         BOOST_WARN_MESSAGE(true, error.what());
-        if (strstr(error.what(), "HIP error: hipInit() failed") != nullptr)
+        if (strstr(error.what(), "HIP Error: could not get device") != nullptr)
             throw HIPInitException(error.what());
         else
             throw error;