Merge pull request #4633 from kjetilly/cuistl_base_utils

Path to multiGPU: Basic utilities for cuISTL

CMakeLists.txt

@@ -541,6 +541,15 @@ if(CUDA_FOUND)
   target_link_libraries( opmsimulators PUBLIC ${CUDA_cusparse_LIBRARY} )
   target_link_libraries( opmsimulators PUBLIC ${CUDA_cublas_LIBRARY} )
   set_tests_properties(cusparseSolver PROPERTIES LABELS gpu_cuda)
+
+  # CUISTL
+  set_tests_properties(cusparse_safe_call
+                       cublas_safe_call
+                       cuda_safe_call
+                       cuda_check_last_error
+                       cublas_handle
+                       cusparse_handle
+                       PROPERTIES LABELS gpu_cuda)
 endif()
 
 if(OpenCL_FOUND)

CMakeLists_files.cmake

@@ -143,6 +143,19 @@ endif()
 if(CUDA_FOUND)
   list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/cuda/cusparseSolverBackend.cu)
   list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/cuda/cuWellContributions.cu)
+
+  # CUISTL SOURCE
+  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/detail/CuBlasHandle.cpp)
+  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/detail/CuSparseHandle.cpp)
+
+  # CUISTL HEADERS
+  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp)
+  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp)
+  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp)
+  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cuda_check_last_error.hpp)
+  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/CuBlasHandle.hpp)
+  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/CuSparseHandle.hpp)
+
 endif()
 if(OPENCL_FOUND)
   list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/BlockedMatrix.cpp)
@@ -220,6 +233,14 @@ if(MPI_FOUND)
 endif()
 if(CUDA_FOUND)
   list(APPEND TEST_SOURCE_FILES tests/test_cusparseSolver.cpp)
+  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cusparse_safe_call.cpp)
+  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cublas_safe_call.cpp)
+  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuda_safe_call.cpp)
+  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuda_check_last_error.cpp)
+  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cublas_handle.cpp)
+  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cusparse_handle.cpp)
+
+
 endif()
 if(OPENCL_FOUND)
   list(APPEND TEST_SOURCE_FILES tests/test_openclSolver.cpp)

opm/simulators/linalg/cuistl/detail/CuBlasHandle.cpp

@@ -0,0 +1,49 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <cublas_v2.h>
+#include <opm/simulators/linalg/cuistl/detail/CuBlasHandle.hpp>
+#include <opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp>
+namespace Opm::cuistl::detail
+{
+
+
+CuBlasHandle::CuBlasHandle()
+{
+    OPM_CUBLAS_SAFE_CALL(cublasCreate(&m_handle));
+}
+
+CuBlasHandle::~CuBlasHandle()
+{
+    OPM_CUBLAS_WARN_IF_ERROR(cublasDestroy(m_handle));
+}
+
+cublasHandle_t
+CuBlasHandle::get()
+{
+    return m_handle;
+}
+
+CuBlasHandle&
+CuBlasHandle::getInstance()
+{
+    static CuBlasHandle instance;
+    return instance;
+}
+
+} // namespace Opm::cuistl::detail

opm/simulators/linalg/cuistl/detail/CuBlasHandle.hpp

@@ -0,0 +1,68 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef OPM_CUBLASHANDLE_HPP
+#define OPM_CUBLASHANDLE_HPP
+#include <cublas_v2.h>
+#include <memory>
+
+namespace Opm::cuistl::detail
+{
+
+/**
+ * @brief The CuBlasHandle class provides a singleton for the simulator universal cuBlasHandle.
+ *
+ * Example use:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/CuBlasHandle.hpp>
+ * void someFunction() {
+ *     auto& cublasHandle = ::Opm::cuistl::detail::CuBlasHandle::getInstance();
+ *     int cuBlasVersion = -1;
+ *     OPM_CUBLAS_SAFE_CALL(cublasGetVersion(cublasHandle.get(), &cuBlasVersion));
+ * }
+ * @endcode
+ *
+ */
+class CuBlasHandle
+{
+public:
+    // This should not be copyable.
+    CuBlasHandle(const CuBlasHandle&) = delete;
+    CuBlasHandle& operator=(const CuBlasHandle&) = delete;
+
+    /**
+     * Calls cublasDestroy() on the handle
+     */
+    ~CuBlasHandle();
+
+    /**
+     * @brief get returns the underlying cuBlas handle (to be used in calls to cublas)
+     */
+    cublasHandle_t get();
+
+    /**
+     * @brief getInstance creates (if necessary) and returns the single unique instance of CuBlasHandle (singleton)
+     */
+    static CuBlasHandle& getInstance();
+
+private:
+    CuBlasHandle();
+    cublasHandle_t m_handle;
+};
+} // namespace Opm::cuistl::detail
+#endif // OPM_CUBLASHANDLE_HPP

opm/simulators/linalg/cuistl/detail/CuSparseHandle.cpp

@@ -0,0 +1,49 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <opm/simulators/linalg/cuistl/detail/CuSparseHandle.hpp>
+#include <opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp>
+namespace Opm::cuistl::detail
+{
+
+
+CuSparseHandle::CuSparseHandle()
+{
+    OPM_CUSPARSE_SAFE_CALL(cusparseCreate(&m_handle));
+    OPM_CUSPARSE_SAFE_CALL(cusparseSetStream(m_handle, 0));
+}
+
+CuSparseHandle::~CuSparseHandle()
+{
+    OPM_CUSPARSE_WARN_IF_ERROR(cusparseDestroy(m_handle));
+}
+
+cusparseHandle_t
+CuSparseHandle::get()
+{
+    return m_handle;
+}
+
+CuSparseHandle&
+CuSparseHandle::getInstance()
+{
+    static CuSparseHandle instance;
+    return instance;
+}
+
+} // namespace Opm::cuistl::detail

opm/simulators/linalg/cuistl/detail/CuSparseHandle.hpp

@@ -0,0 +1,67 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef OPM_CUSPARSEHANDLE_HPP
+#define OPM_CUSPARSEHANDLE_HPP
+#include <cusparse.h>
+#include <memory>
+
+namespace Opm::cuistl::detail
+{
+
+/**
+ * @brief The CuSparseHandle class provides a singleton for the simulator universal cuSparseHandle.
+ *
+ * Example use:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/CuSparseHandle.hpp>
+ * void someFunction() {
+ *     auto& cuSparseHandle = ::Opm::cuistl::detail::CuSparseHandle::getInstance();
+ *     int cuSparseVersion = -1;
+ *     OPM_CUSPARSE_SAFE_CALL(cusparseGetVersion(cuSparseHandle.get(), &cuSparseVersion));
+ * }
+ * @endcode
+ */
+class CuSparseHandle
+{
+public:
+    // This should not be copyable.
+    CuSparseHandle(const CuSparseHandle&) = delete;
+    CuSparseHandle& operator=(const CuSparseHandle&) = delete;
+
+    /**
+     * Calls cuSparseDestroy on the handle
+     */
+    ~CuSparseHandle();
+
+    /**
+     * @brief get returns the underlying cuSparse handle (to be used in calls to cusparse)
+     */
+    cusparseHandle_t get();
+
+    /**
+     * @brief getInstance creates (if necessary) and returns the single unique instance of CuSparseHandle (singleton)
+     */
+    static CuSparseHandle& getInstance();
+
+private:
+    CuSparseHandle();
+    cusparseHandle_t m_handle;
+};
+} // namespace Opm::cuistl::detail
+#endif // OPM_CUSPARSEHANDLE_HPP

opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp

@@ -0,0 +1,219 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef OPM_CUBLAS_SAFE_CALL_HPP
+#define OPM_CUBLAS_SAFE_CALL_HPP
+#include <cublas_v2.h>
+#include <exception>
+#include <fmt/core.h>
+#include <opm/common/ErrorMacros.hpp>
+#include <opm/common/OpmLog/OpmLog.hpp>
+#include <string_view>
+
+
+
+
+namespace Opm::cuistl::detail
+{
+
+#define CHECK_CUBLAS_ERROR_TYPE(code, x)                                                                               \
+    if (code == x) {                                                                                                   \
+        return #x;                                                                                                     \
+    }
+
+namespace
+{
+    /**
+     * @brief getCublasErrorCodeToString Converts an error code returned from a cublas function a human readable string.
+     * @param code an error code from a cublas routine
+     * @return a human readable string.
+     */
+    inline std::string getCublasErrorCodeToString(int code)
+    {
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_SUCCESS);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_NOT_INITIALIZED);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_ALLOC_FAILED);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_INVALID_VALUE);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_ARCH_MISMATCH);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_MAPPING_ERROR);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_EXECUTION_FAILED);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_INTERNAL_ERROR);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_NOT_SUPPORTED);
+        CHECK_CUBLAS_ERROR_TYPE(code, CUBLAS_STATUS_LICENSE_ERROR);
+
+        return fmt::format("UNKNOWN CUBLAS ERROR {}.", code);
+    }
+
+#undef CHECK_CUBLAS_ERROR_TYPE
+
+} // namespace
+
+/**
+ * @brief getCublasErrorMessage generates the error message to display for a given error.
+ *
+ * @param error the error code from cublas
+ * @param expression the expresison (say "cublasCreate(&handle)")
+ * @param filename the code file the error occured in (typically __FILE__)
+ * @param functionName name of the function the error occured in (typically __func__)
+ * @param lineNumber the line number the error occured in (typically __LINE__)
+ *
+ * @todo Refactor to use std::source_location once we shift to C++20
+ *
+ * @return An error message to be displayed.
+ *
+ * @note This function is mostly for internal use.
+ */
+inline std::string
+getCublasErrorMessage(cublasStatus_t error,
+                      const std::string_view& expression,
+                      const std::string_view& filename,
+                      const std::string_view& functionName,
+                      size_t lineNumber)
+{
+    return fmt::format("cuBLAS expression did not execute correctly. Expression was: \n\n"
+                       "    {}\n\n"
+                       "in function {}, in {}, at line {}.\n"
+                       "CuBLAS error code was: {}\n",
+                       expression,
+                       functionName,
+                       filename,
+                       lineNumber,
+                       getCublasErrorCodeToString(error));
+}
+
+/**
+ * @brief cublasSafeCall checks the return type of the CUBLAS expression (function call) and throws an exception if it
+ * does not equal CUBLAS_STATUS_SUCCESS.
+ *
+ * @param error the error code from cublas
+ * @param expression the expresison (say "cublasCreate(&handle)")
+ * @param filename the code file the error occured in (typically __FILE__)
+ * @param functionName name of the function the error occured in (typically __func__)
+ * @param lineNumber the line number the error occured in (typically __LINE__)
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp>
+ * #include <cublas_v2.h>
+ *
+ * void some_function() {
+ *     cublasHandle_t cublasHandle;
+ *     cudaSafeCall(cublasCreate(&cublasHandle), "cublasCreate(&cublasHandle)", __FILE__, __func__, __LINE__);
+ * }
+ * @endcode
+ *
+ * @note It is probably easier to use the macro OPM_CUBLAS_SAFE_CALL
+ *
+ * @todo Refactor to use std::source_location once we shift to C++20
+ */
+inline void
+cublasSafeCall(cublasStatus_t error,
+               const std::string_view& expression,
+               const std::string_view& filename,
+               const std::string_view& functionName,
+               size_t lineNumber)
+{
+    if (error != CUBLAS_STATUS_SUCCESS) {
+        OPM_THROW(std::runtime_error, getCublasErrorMessage(error, expression, filename, functionName, lineNumber));
+    }
+}
+
+/**
+ * @brief cublasWarnIfError checks the return type of the CUBLAS expression (function call) and issues a warning if it
+ * does not equal CUBLAS_STATUS_SUCCESS.
+ *
+ * @param error the error code from cublas
+ * @param expression the expresison (say "cublasCreate(&handle)")
+ * @param filename the code file the error occured in (typically __FILE__)
+ * @param functionName name of the function the error occured in (typically __func__)
+ * @param lineNumber the line number the error occured in (typically __LINE__)
+ *
+ * @return the error sent in (for convenience).
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp>
+ * #include <cublas_v2.h>
+ *
+ * void some_function() {
+ *     cublasHandle_t cublasHandle;
+ *     cublasWarnIfError(cublasCreate(&cublasHandle), "cublasCreate(&cublasHandle)", __FILE__, __func__, __LINE__);
+ * }
+ * @endcode
+ *
+ * @note It is probably easier to use the macro OPM_CUBLAS_WARN_IF_ERROR
+ * @note Prefer the cublasSafeCall/OPM_CUBLAS_SAFE_CALL counterpart unless you really don't want to throw an exception.
+ *
+ * @todo Refactor to use std::source_location once we shift to C++20
+ */
+inline cublasStatus_t
+cublasWarnIfError(cublasStatus_t error,
+                  const std::string_view& expression,
+                  const std::string_view& filename,
+                  const std::string_view& functionName,
+                  size_t lineNumber)
+{
+    if (error != CUBLAS_STATUS_SUCCESS) {
+        OpmLog::warning(getCublasErrorMessage(error, expression, filename, functionName, lineNumber));
+    }
+
+    return error;
+}
+} // namespace Opm::cuistl::detail
+
+/**
+ * @brief OPM_CUBLAS_SAFE_CALL checks the return type of the cublas expression (function call) and throws an exception
+ * if it does not equal CUBLAS_STATUS_SUCCESS.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <cublas_v2.h>
+ * #include <opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp>
+ *
+ * void some_function() {
+ *     cublasHandle_t cublasHandle;
+ *     OPM_CUBLAS_SAFE_CALL(cublasCreate(&cublasHandle));
+ * }
+ * @endcode
+ *
+ * @note This should be used for any call to cuBlas unless you have a good reason not to.
+ */
+#define OPM_CUBLAS_SAFE_CALL(expression)                                                                               \
+    ::Opm::cuistl::detail::cublasSafeCall(expression, #expression, __FILE__, __func__, __LINE__)
+
+/**
+ * @brief OPM_CUBLAS_WARN_IF_ERROR checks the return type of the cublas expression (function call) and issues a warning
+ * if it does not equal CUBLAS_STATUS_SUCCESS.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <cublas_v2.h>
+ * #include <opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp>
+ *
+ * void some_function() {
+ *     cublasHandle_t cublasHandle;
+ *     OPM_CUBLAS_WARN_IF_ERROR(cublasCreate(&cublasHandle));
+ * }
+ * @endcode
+ *
+ * @note Prefer the cublasSafeCall/OPM_CUBLAS_SAFE_CALL counterpart unless you really don't want to throw an exception.
+ */
+#define OPM_CUBLAS_WARN_IF_ERROR(expression)                                                                           \
+    ::Opm::cuistl::detail::cublasWarnIfError(expression, #expression, __FILE__, __func__, __LINE__)
+
+#endif // OPM_CUBLAS_SAFE_CALL_HPP

opm/simulators/linalg/cuistl/detail/cuda_check_last_error.hpp

@@ -0,0 +1,105 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef OPM_CUDA_CHECK_LAST_ERROR_HPP
+#define OPM_CUDA_CHECK_LAST_ERROR_HPP
+#include <cuda_runtime.h>
+#include <fmt/core.h>
+#include <opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp>
+
+/**
+ * @brief OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE checks the return type of cudaDeviceSynchronize(),
+ * and throws an exception if cudaDeviceSynchronize() does not equal cudaSuccess.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cuda_check_last_error.hpp>
+ *
+ * void some_function() {
+ *     OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE;
+ * }
+ * @endcode
+ *
+ * @note This can be used to debug the code, or simply make sure that no error has occured.
+ * @note This is a rather heavy operation, so prefer to use only in Debug mode (see OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE_IF_DEBUG)
+ */
+#define OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE OPM_CUDA_SAFE_CALL(cudaDeviceSynchronize())
+
+#ifdef NDEBUG
+#define OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE_IF_DEBUG
+#else
+
+/**
+ * @brief OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE_IF_DEBUG checks the return type of cudaDeviceSynchronize only if NDEBUG is not defined,
+ * and throws an exception if cudaDeviceSynchronize() does not equal cudaSuccess.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp>
+ *
+ * void some_function() {
+ *     OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE_IF_DEBUG;
+ * }
+ * @endcode
+ *
+ * @note This can be used to debug the code, or simply make sure that no error has occured.
+ */
+#define OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE_IF_DEBUG OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE
+#endif
+
+
+/**
+ * @brief OPM_CUDA_CHECK_LAST_ERROR checks the return type of cudaGetLastError(),
+ * and throws an exception if cudaGetLastError() does not equal cudaSuccess.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cuda_check_last_error.hpp>
+ *
+ * void some_function() {
+ *     OPM_CUDA_CHECK_LAST_ERROR;
+ * }
+ * @endcode
+ *
+ * @note This can be used to debug the code, or simply make sure that no error has occured.
+ */
+#define OPM_CUDA_CHECK_LAST_ERROR OPM_CUDA_SAFE_CALL(cudaGetLastError())
+
+#ifdef NDEBUG
+#define OPM_CUDA_CHECK_LAST_ERROR_IF_DEBUG
+#else
+
+/**
+ * @brief OPM_CUDA_CHECK_LAST_ERROR_IF_DEBUG checks the return type of cudaGetLastError() only if NDEBUG is not defined,
+ * and throws an exception if cudaGetLastError() does not equal cudaSuccess.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cuda_check_last_error.hpp>
+ *
+ * void some_function() {
+ *     OPM_CUDA_CHECK_LAST_ERROR_IF_DEBUG;
+ * }
+ * @endcode
+ *
+ * @note This can be used to debug the code, or simply make sure that no error has occured.
+ */
+#define OPM_CUDA_CHECK_LAST_ERROR_IF_DEBUG OPM_CUDA_CHECK_LAST_ERROR
+#endif
+
+#endif

opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp

@@ -0,0 +1,178 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef OPM_CUDA_SAFE_CALL_HPP
+#define OPM_CUDA_SAFE_CALL_HPP
+#include <cuda_runtime.h>
+#include <fmt/core.h>
+#include <opm/common/ErrorMacros.hpp>
+#include <opm/common/OpmLog/OpmLog.hpp>
+#include <string_view>
+
+namespace Opm::cuistl::detail
+{
+/**
+ * @brief getCudaErrorMessage generates the error message to display for a given error.
+ *
+ * @param error the error code from cublas
+ * @param expression the expresison (say "cudaMalloc(&pointer, 1)")
+ * @param filename the code file the error occured in (typically __FILE__)
+ * @param functionName name of the function the error occured in (typically __func__)
+ * @param lineNumber the line number the error occured in (typically __LINE__)
+ *
+ * @todo Refactor to use std::source_location once we shift to C++20
+ *
+ * @return An error message to be displayed.
+ *
+ * @note This function is mostly for internal use.
+ */
+inline std::string
+getCudaErrorMessage(cudaError_t error,
+                    const std::string_view& expression,
+                    const std::string_view& filename,
+                    const std::string_view& functionName,
+                    size_t lineNumber)
+{
+    return fmt::format("CUDA expression did not execute correctly. Expression was: \n"
+                       "    {}\n"
+                       "CUDA error was {}\n"
+                       "in function {}, in {}, at line {}\n",
+                       expression,
+                       cudaGetErrorString(error),
+                       functionName,
+                       filename,
+                       lineNumber);
+}
+
+/**
+ * @brief cudaSafeCall checks the return type of the CUDA expression (function call) and throws an exception if it
+ * does not equal cudaSuccess.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp>
+ * #include <cuda_runtime.h>
+ *
+ * void some_function() {
+ *     void* somePointer;
+ *     cudaSafeCall(cudaMalloc(&somePointer, 1), "cudaMalloc(&somePointer, 1)", __FILE__, __func__, __LINE__);
+ * }
+ * @endcode
+ *
+ * @note It is probably easier to use the macro OPM_CUDA_SAFE_CALL
+ *
+ * @todo Refactor to use std::source_location once we shift to C++20
+ */
+inline void
+cudaSafeCall(cudaError_t error,
+             const std::string_view& expression,
+             const std::string_view& filename,
+             const std::string_view& functionName,
+             size_t lineNumber)
+{
+    if (error != cudaSuccess) {
+        OPM_THROW(std::runtime_error, getCudaErrorMessage(error, expression, filename, functionName, lineNumber));
+    }
+}
+
+/**
+ * @brief cudaWarnIfError checks the return type of the CUDA expression (function call) and issues a warning if it
+ * does not equal cudaSuccess.
+ *
+ * @param error the error code from cublas
+ * @param expression the expresison (say "cudaMalloc(&pointer, 1)")
+ * @param filename the code file the error occured in (typically __FILE__)
+ * @param functionName name of the function the error occured in (typically __func__)
+ * @param lineNumber the line number the error occured in (typically __LINE__)
+ *
+ * @return the error sent in (for convenience).
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp>
+ * #include <cuda_runtime.h>
+ *
+ * void some_function() {
+ *     void* somePointer;
+ *     cudaWarnIfError(cudaMalloc(&somePointer, 1), "cudaMalloc(&somePointer, 1)", __FILE__, __func__, __LINE__);
+ * }
+ * @endcode
+ *
+ * @note It is probably easier to use the macro OPM_CUDA_WARN_IF_ERROR
+ *
+ * @note Prefer the cudaSafeCall/OPM_CUDA_SAFE_CALL counterpart unless you really don't want to throw an exception.
+ *
+ * @todo Refactor to use std::source_location once we shift to C++20
+ */
+inline cudaError_t
+cudaWarnIfError(cudaError_t error,
+                const std::string_view& expression,
+                const std::string_view& filename,
+                const std::string_view& functionName,
+                size_t lineNumber)
+{
+    if (error != cudaSuccess) {
+        OpmLog::warning(getCudaErrorMessage(error, expression, filename, functionName, lineNumber));
+    }
+
+    return error;
+}
+} // namespace Opm::cuistl::detail
+
+/**
+ * @brief OPM_CUDA_SAFE_CALL checks the return type of the CUDA expression (function call) and throws an exception if it
+ * does not equal cudaSuccess.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp>
+ * #include <cuda_runtime.h>
+ *
+ * void some_function() {
+ *     void* somePointer;
+ *     OPM_CUDA_SAFE_CALL(cudaMalloc(&somePointer, 1));
+ * }
+ * @endcode
+ *
+ * @note This should be used for any call to the CUDA runtime API unless you have a good reason not to.
+ */
+#define OPM_CUDA_SAFE_CALL(expression)                                                                                 \
+    ::Opm::cuistl::detail::cudaSafeCall(expression, #expression, __FILE__, __func__, __LINE__)
+
+
+/**
+ * @brief OPM_CUDA_WARN_IF_ERROR checks the return type of the CUDA expression (function call) and issues a warning if
+ * it does not equal cudaSuccess.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp>
+ * #include <cuda_runtime.h>
+ *
+ * void some_function() {
+ *     void* somePointer;
+ *     OPM_CUDA_WARN_IF_ERROR(cudaMalloc(&somePointer, 1));
+ * }
+ * @endcode
+ *
+ * @note Prefer the cudaSafeCall/OPM_CUDA_SAFE_CALL counterpart unless you really don't want to throw an exception.
+ */
+#define OPM_CUDA_WARN_IF_ERROR(expression)                                                                             \
+    ::Opm::cuistl::detail::cudaWarnIfError(expression, #expression, __FILE__, __func__, __LINE__)
+
+#endif

opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp

@@ -0,0 +1,208 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef OPM_CUSPARSE_SAFE_CALL_HPP
+#define OPM_CUSPARSE_SAFE_CALL_HPP
+#include <cusparse.h>
+#include <exception>
+#include <fmt/core.h>
+#include <opm/common/ErrorMacros.hpp>
+#include <opm/common/OpmLog/OpmLog.hpp>
+
+namespace Opm::cuistl::detail
+{
+
+#define CHECK_CUSPARSE_ERROR_TYPE(code, x)                                                                             \
+    if (code == x) {                                                                                                   \
+        return #x;                                                                                                     \
+    }
+/**
+ * @brief getCusparseErrorCodeToString Converts an error code returned from a cusparse function a human readable string.
+ * @param code an error code from a cusparse routine
+ * @return a human readable string.
+ */
+inline std::string
+getCusparseErrorCodeToString(int code)
+{
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_SUCCESS);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_NOT_INITIALIZED);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_ALLOC_FAILED);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_INVALID_VALUE);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_ARCH_MISMATCH);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_MAPPING_ERROR);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_EXECUTION_FAILED);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_INTERNAL_ERROR);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_ZERO_PIVOT);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_NOT_SUPPORTED);
+    CHECK_CUSPARSE_ERROR_TYPE(code, CUSPARSE_STATUS_INSUFFICIENT_RESOURCES);
+    return fmt::format("UNKNOWN CUSPARSE ERROR {}.", code);
+}
+
+#undef CHECK_CUSPARSE_ERROR_TYPE
+/**
+ * @brief getCusparseErrorMessage generates the error message to display for a given error.
+ *
+ * @param error the error code from cublas
+ * @param expression the expresison (say "cusparseCreate(&handle)")
+ * @param filename the code file the error occured in (typically __FILE__)
+ * @param functionName name of the function the error occured in (typically __func__)
+ * @param lineNumber the line number the error occured in (typically __LINE__)
+ *
+ * @todo Refactor to use std::source_location once we shift to C++20
+ *
+ * @return An error message to be displayed.
+ *
+ * @note This function is mostly for internal use.
+ */
+inline std::string
+getCusparseErrorMessage(cusparseStatus_t error,
+                        const std::string_view& expression,
+                        const std::string_view& filename,
+                        const std::string_view& functionName,
+                        size_t lineNumber)
+{
+    return fmt::format("cuSparse expression did not execute correctly. Expression was: \n\n"
+                       "    {}\n\nin function {}, in {}, at line {}\n"
+                       "CuSparse error code was: {}\n",
+                       expression,
+                       functionName,
+                       filename,
+                       lineNumber,
+                       getCusparseErrorCodeToString(error));
+}
+
+/**
+ * @brief cusparseSafeCall checks the return type of the CUSPARSE expression (function call) and throws an exception if
+ * it does not equal CUSPARSE_STATUS_SUCCESS.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp>
+ * #include <cublas_v2.h>
+ *
+ * void some_function() {
+ *     cusparseHandle_t cusparseHandle;
+ *     cusparseSafeCall(cusparseCreate(&cusparseHandle), "cusparseCreate(&cusparseHandle)", __FILE__, __func__,
+ * __LINE__);
+ * }
+ * @endcode
+ *
+ * @note It is probably easier to use the macro OPM_CUBLAS_SAFE_CALL
+ *
+ * @todo Refactor to use std::source_location once we shift to C++20
+ */
+inline void
+cusparseSafeCall(cusparseStatus_t error,
+                 const std::string_view& expression,
+                 const std::string_view& filename,
+                 const std::string_view& functionName,
+                 size_t lineNumber)
+{
+    if (error != CUSPARSE_STATUS_SUCCESS) {
+        OPM_THROW(std::runtime_error, getCusparseErrorMessage(error, expression, filename, functionName, lineNumber));
+    }
+}
+
+/**
+ * @brief cusparseWarnIfError checks the return type of the CUSPARSE expression (function call) and issues a warning if
+ * it does not equal CUSPARSE_STATUS_SUCCESS.
+ *
+ * @param error the error code from cublas
+ * @param expression the expresison (say "cublasCreate(&handle)")
+ * @param filename the code file the error occured in (typically __FILE__)
+ * @param functionName name of the function the error occured in (typically __func__)
+ * @param lineNumber the line number the error occured in (typically __LINE__)
+ *
+ * @return the error sent in (for convenience).
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp>
+ * #include <cublas_v2.h>
+ *
+ * void some_function() {
+ *     cusparseHandle_t cusparseHandle;
+ *     cusparseWarnIfError(cusparseCreate(&cusparseHandle), "cusparseCreate(&cusparseHandle)", __FILE__, __func__,
+ * __LINE__);
+ * }
+ * @endcode
+ *
+ * @note It is probably easier to use the macro OPM_CUSPARSE_WARN_IF_ERROR
+ * @note Prefer the cusparseSafeCall/OPM_CUSPARSE_SAFE_CALL counterpart unless you really don't want to throw an
+ * exception.
+ * @todo Refactor to use std::source_location once we shift to C++20
+ */
+inline cusparseStatus_t
+cusparseWarnIfError(cusparseStatus_t error,
+                    const std::string_view& expression,
+                    const std::string_view& filename,
+                    const std::string_view& functionName,
+                    size_t lineNumber)
+{
+    if (error != CUSPARSE_STATUS_SUCCESS) {
+        OpmLog::warning(getCusparseErrorMessage(error, expression, filename, functionName, lineNumber));
+    }
+
+    return error;
+}
+} // namespace Opm::cuistl::detail
+
+
+
+/**
+ * @brief OPM_CUSPARSE_SAFE_CALL checks the return type of the cusparse expression (function call) and throws an
+ * exception if it does not equal CUSPARSE_STATUS_SUCCESS.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp>
+ * #include <cusparse.h>
+ *
+ * void some_function() {
+ *     cusparseHandle_t cusparseHandle;
+ *     OPM_CUSPARSE_SAFE_CALL(cusparseCreate(&cusparseHandle));
+ * }
+ * @endcode
+ *
+ * @note This should be used for any call to cuSparse unless you have a good reason not to.
+ */
+#define OPM_CUSPARSE_SAFE_CALL(expression)                                                                             \
+    ::Opm::cuistl::detail::cusparseSafeCall(expression, #expression, __FILE__, __func__, __LINE__)
+
+/**
+ * @brief OPM_CUSPARSE_WARN_IF_ERROR checks the return type of the cusparse expression (function call) and issues a
+ * warning if it does not equal CUSPARSE_STATUS_SUCCESS.
+ *
+ * Example usage:
+ * @code{.cpp}
+ * #include <opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp>
+ * #include <cusparse.h>
+ *
+ * void some_function() {
+ *     cusparseHandle_t cusparseHandle;
+ *     OPM_CUSPARSE_WARN_IF_ERROR(cusparseCreate(&cusparseHandle));
+ * }
+ * @endcode
+ *
+ * @note Prefer the cusparseSafeCall/OPM_CUBLAS_SAFE_CALL counterpart unless you really don't want to throw an
+ * exception.
+ */
+#define OPM_CUSPARSE_WARN_IF_ERROR(expression)                                                                         \
+    ::Opm::cuistl::detail::cusparseWarnIfError(expression, #expression, __FILE__, __func__, __LINE__)
+#endif // OPM_CUSPARSE_SAFE_CALL_HPP

tests/cuistl/test_cublas_handle.cpp

@@ -0,0 +1,34 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp"
+#include <config.h>
+
+#define BOOST_TEST_MODULE TestCublasHandle
+
+#include <boost/test/unit_test.hpp>
+#include <opm/simulators/linalg/cuistl/detail/CuBlasHandle.hpp>
+
+BOOST_AUTO_TEST_CASE(TestGetCublasVersion)
+{
+    auto& cublasHandle = ::Opm::cuistl::detail::CuBlasHandle::getInstance();
+    int cuBlasVersion = -1;
+    OPM_CUBLAS_SAFE_CALL(cublasGetVersion(cublasHandle.get(), &cuBlasVersion));
+
+    BOOST_CHECK_LT(0, cuBlasVersion);
+}

tests/cuistl/test_cublas_safe_call.cpp

@@ -0,0 +1,47 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <config.h>
+
+#define BOOST_TEST_MODULE TestCublasSafeCall
+
+#include <boost/test/unit_test.hpp>
+#include <cublas_v2.h>
+#include <opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp>
+
+BOOST_AUTO_TEST_CASE(TestCreateHandle)
+{
+    cublasHandle_t cublasHandle;
+    BOOST_CHECK_NO_THROW(OPM_CUBLAS_SAFE_CALL(cublasCreate(&cublasHandle)););
+}
+
+BOOST_AUTO_TEST_CASE(TestThrows)
+{
+    std::vector<cublasStatus_t> errorCodes {{CUBLAS_STATUS_NOT_INITIALIZED,
+                                             CUBLAS_STATUS_ALLOC_FAILED,
+                                             CUBLAS_STATUS_INVALID_VALUE,
+                                             CUBLAS_STATUS_ARCH_MISMATCH,
+                                             CUBLAS_STATUS_MAPPING_ERROR,
+                                             CUBLAS_STATUS_EXECUTION_FAILED,
+                                             CUBLAS_STATUS_INTERNAL_ERROR,
+                                             CUBLAS_STATUS_NOT_SUPPORTED,
+                                             CUBLAS_STATUS_LICENSE_ERROR}};
+    for (auto code : errorCodes) {
+        BOOST_CHECK_THROW(OPM_CUBLAS_SAFE_CALL(code), std::exception);
+    }
+}

tests/cuistl/test_cuda_check_last_error.cpp

@@ -0,0 +1,38 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <config.h>
+
+#define BOOST_TEST_MODULE TestCudaCheckLastError
+
+#include <boost/test/unit_test.hpp>
+#include <opm/simulators/linalg/cuistl/detail/cuda_check_last_error.hpp>
+
+
+BOOST_AUTO_TEST_CASE(TestNoThrowLastError)
+{
+    BOOST_CHECK_NO_THROW(OPM_CUDA_CHECK_LAST_ERROR;);
+    BOOST_CHECK_NO_THROW(OPM_CUDA_CHECK_LAST_ERROR_IF_DEBUG;);
+}
+
+
+BOOST_AUTO_TEST_CASE(TestNoThrowDeviceSynchronize)
+{
+    BOOST_CHECK_NO_THROW(OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE;);
+    BOOST_CHECK_NO_THROW(OPM_CUDA_CHECK_DEVICE_SYNCHRONIZE_IF_DEBUG;);
+}

tests/cuistl/test_cuda_safe_call.cpp

@@ -0,0 +1,40 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <config.h>
+
+#define BOOST_TEST_MODULE TestCudaSafeCall
+#include <boost/test/unit_test.hpp>
+#include <cuda_runtime.h>
+#include <opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp>
+
+BOOST_AUTO_TEST_CASE(TestCudaMalloc)
+{
+    void* pointer;
+    BOOST_CHECK_NO_THROW(OPM_CUDA_SAFE_CALL(cudaMalloc(&pointer, 1)););
+}
+
+
+BOOST_AUTO_TEST_CASE(TestThrows)
+{
+    // Just testing a subset here.
+    std::vector<cudaError_t> errorCodes {{cudaErrorAddressOfConstant, cudaErrorAlreadyAcquired}};
+    for (auto code : errorCodes) {
+        BOOST_CHECK_THROW(OPM_CUDA_SAFE_CALL(code), std::exception);
+    }
+}

tests/cuistl/test_cusparse_handle.cpp

@@ -0,0 +1,33 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <config.h>
+
+#define BOOST_TEST_MODULE TestSparseHandle
+
+#include <boost/test/unit_test.hpp>
+#include <opm/simulators/linalg/cuistl/detail/CuSparseHandle.hpp>
+#include <opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp>
+
+BOOST_AUTO_TEST_CASE(TestGetSparseVersion)
+{
+    auto& cuSparseHandle = ::Opm::cuistl::detail::CuSparseHandle::getInstance();
+    int cuSparseVersion = -1;
+    OPM_CUSPARSE_SAFE_CALL(cusparseGetVersion(cuSparseHandle.get(), &cuSparseVersion));
+    BOOST_CHECK_LT(0, cuSparseVersion);
+}

tests/cuistl/test_cusparse_safe_call.cpp

@@ -0,0 +1,49 @@
+/*
+  Copyright 2022-2023 SINTEF AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <config.h>
+
+#define BOOST_TEST_MODULE TestCusparseSafeCall
+
+#include <boost/test/unit_test.hpp>
+#include <cusparse.h>
+#include <opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp>
+
+BOOST_AUTO_TEST_CASE(TestCreateHandle)
+{
+    cusparseHandle_t cusparseHandle;
+    BOOST_CHECK_NO_THROW(OPM_CUSPARSE_SAFE_CALL(cusparseCreate(&cusparseHandle)););
+}
+
+BOOST_AUTO_TEST_CASE(TestThrows)
+{
+    std::vector<cusparseStatus_t> errorCodes {{CUSPARSE_STATUS_NOT_INITIALIZED,
+                                               CUSPARSE_STATUS_ALLOC_FAILED,
+                                               CUSPARSE_STATUS_INVALID_VALUE,
+                                               CUSPARSE_STATUS_ARCH_MISMATCH,
+                                               CUSPARSE_STATUS_MAPPING_ERROR,
+                                               CUSPARSE_STATUS_EXECUTION_FAILED,
+                                               CUSPARSE_STATUS_INTERNAL_ERROR,
+                                               CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED,
+                                               CUSPARSE_STATUS_ZERO_PIVOT,
+                                               CUSPARSE_STATUS_NOT_SUPPORTED,
+                                               CUSPARSE_STATUS_INSUFFICIENT_RESOURCES}};
+    for (auto code : errorCodes) {
+        BOOST_CHECK_THROW(OPM_CUSPARSE_SAFE_CALL(code), std::exception);
+    }
+}