diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake
index af6e53c5b3..e16173f05d 100644
--- a/CMakeLists_files.cmake
+++ b/CMakeLists_files.cmake
@@ -583,7 +583,9 @@ if(CUDA_FOUND OR hip_FOUND)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_GpuPressureTransferPolicy.cpp)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_deviceBlockOperations.cu)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_primaryvarswithdifferentvector.cpp)
-  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_primary_variables_gpu.cu)
+  if(hip_FOUND OR CUDA_VERSION VERSION_GREATER 12.0)
+    ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_primary_variables_gpu.cu)
+  endif()
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_MiniMatrix.cu)
   # Boost < 1.75 + nvcc = trouble in this test
   if(Boost_VERSION VERSION_GREATER 1.74)
diff --git a/opm/simulators/linalg/gpuistl/detail/gpu_safe_call.hpp b/opm/simulators/linalg/gpuistl/detail/gpu_safe_call.hpp
index 97c4015b02..50d0c1d8f6 100644
--- a/opm/simulators/linalg/gpuistl/detail/gpu_safe_call.hpp
+++ b/opm/simulators/linalg/gpuistl/detail/gpu_safe_call.hpp
@@ -19,7 +19,11 @@
 #ifndef OPM_GPU_SAFE_CALL_HPP
 #define OPM_GPU_SAFE_CALL_HPP
 #include <cuda_runtime.h>
+#if CUDA_VERSION >= 12100
 #include <fmt/core.h>
+#else
+#include <sstream>
+#endif
 #include <opm/common/ErrorMacros.hpp>
 #include <opm/common/OpmLog/OpmLog.hpp>
 #include <string_view>
@@ -30,10 +34,10 @@ namespace Opm::gpuistl::detail
  * @brief getCudaErrorMessage generates the error message to display for a given error.
  *
  * @param error the error code from cublas
- * @param expression the expresison (say "cudaMalloc(&pointer, 1)")
- * @param filename the code file the error occured in (typically __FILE__)
- * @param functionName name of the function the error occured in (typically __func__)
- * @param lineNumber the line number the error occured in (typically __LINE__)
+ * @param expression the expression (say "cudaMalloc(&pointer, 1)")
+ * @param filename the file the error occurred in (typically __FILE__)
+ * @param functionName name of the function the error occurred in (typically __func__)
+ * @param lineNumber the line number the error occurred in (typically __LINE__)
  *
  * @todo Refactor to use std::source_location once we shift to C++20
  *
@@ -48,6 +52,7 @@ getCudaErrorMessage(cudaError_t error,
                     const std::string_view& functionName,
                     size_t lineNumber)
 {
+#if CUDA_VERSION >= 12100
     return fmt::format(fmt::runtime("GPU expression did not execute correctly. Expression was: \n"
                        "    {}\n"
                        "GPU error was {}\n"
@@ -57,6 +62,15 @@ getCudaErrorMessage(cudaError_t error,
                        functionName,
                        filename,
                        lineNumber);
+#else
+    std::stringstream str;
+    str << "GPU expression did not execute correctly. Expression was: \n"
+        << "    " << expression
+        << "\nGPU error was " << cudaGetErrorString(error)
+        << "\nin function " << functionName << ", in " << filename
+        << ", at line " << lineNumber << '\n';
+    return str.str();
+#endif
 }
 
 /**
diff --git a/opm/simulators/linalg/gpuistl/detail/kernel_enums.hpp b/opm/simulators/linalg/gpuistl/detail/kernel_enums.hpp
index e0acc2056c..96e8d15140 100644
--- a/opm/simulators/linalg/gpuistl/detail/kernel_enums.hpp
+++ b/opm/simulators/linalg/gpuistl/detail/kernel_enums.hpp
@@ -20,8 +20,16 @@
 #ifndef OPM_GPUISTL_KERNEL_ENUMS_HPP
 #define OPM_GPUISTL_KERNEL_ENUMS_HPP
 
+#include <opm/common/ErrorMacros.hpp>
+
 #include <cuda_runtime.h>
 
+#if CUDA_VERSION >= 12100
+#include <fmt/core.h>
+#else
+#include <sstream>
+#endif
+
 /*
     This file organizes a growing amount of different mixed precision options for the preconditioners.
 */
@@ -40,6 +48,7 @@ namespace Opm::gpuistl {
 
     inline MatrixStorageMPScheme makeMatrixStorageMPScheme(int scheme) {
         if (!detail::isValidMatrixStorageMPScheme(scheme)) {
+#if CUDA_VERSION >= 12100
             OPM_THROW(std::invalid_argument,
                       fmt::format(fmt::runtime("Invalid matrix storage mixed precision scheme chosen: {}.\n"
                                   "Valid Schemes:\n"
@@ -47,6 +56,15 @@ namespace Opm::gpuistl {
                                   "\t1: FLOAT_DIAG_FLOAT_OFFDIAG\n"
                                   "\t2: DOUBLE_DIAG_FLOAT_OFFDIAG"),
                                   scheme));
+#else
+            std::stringstream str;
+            str << "Invalid matrix storage mixed precision scheme chosen: " << scheme << ".\n"
+                << "Valid Schemes:\n"
+                   "\t0: DOUBLE_DIAG_DOUBLE_OFFDIAG\n"
+                   "\t1: FLOAT_DIAG_FLOAT_OFFDIAG\n"
+                   "\t2: DOUBLE_DIAG_FLOAT_OFFDIAG";
+            OPM_THROW(std::invalid_argument, str.str());
+#endif
         }
         return static_cast<MatrixStorageMPScheme>(scheme);
     }
diff --git a/opm/simulators/linalg/gpuistl/detail/safe_conversion.hpp b/opm/simulators/linalg/gpuistl/detail/safe_conversion.hpp
index 930231e272..30a928d0bc 100644
--- a/opm/simulators/linalg/gpuistl/detail/safe_conversion.hpp
+++ b/opm/simulators/linalg/gpuistl/detail/safe_conversion.hpp
@@ -19,15 +19,19 @@
 #ifndef OPM_CUISTL_SAFE_CONVERSION_HPP
 #define OPM_CUISTL_SAFE_CONVERSION_HPP
 
-
+#include <opm/common/ErrorMacros.hpp>
 
 #include <cstddef>
-#include <fmt/format.h>
 #include <limits>
-#include <opm/common/ErrorMacros.hpp>
 #include <type_traits>
 #include <cuda_runtime.h>
 
+#if CUDA_VERSION >= 12100
+#include <fmt/format.h>
+#else
+#include <sstream>
+#endif
+
 
 /**
  * Provides various utilities for doing signed to unsigned conversion, unsigned to signed, 32 bits to 64 bits and 64
@@ -64,10 +68,17 @@ to_int(std::size_t s)
 
 
     if (s > std::size_t(std::numeric_limits<int>::max())) {
+#if CUDA_VERSION >= 12100
         OPM_THROW(std::invalid_argument,
                   fmt::format(fmt::runtime("Trying to convert {} to int, but it is out of range. Maximum possible int: {}. "),
                               s,
                               std::numeric_limits<int>::max()));
+#else
+        std::stringstream str;
+        str << "Trying to convert " << s << " to int, but it is out of range. Maximum possible int: "
+            << std::numeric_limits<int>::max() << " .";
+        OPM_THROW(std::invalid_argument, str.str());
+#endif
     }
 
     // We know it will be in range here:
@@ -100,7 +111,13 @@ to_size_t(int i)
     assert(i >= int(0));
 #else
     if (i < int(0)) {
+#if CUDA_VERSION >= 12100
         OPM_THROW(std::invalid_argument, fmt::format(fmt::runtime("Trying to convert the negative number {} to size_t."), i));
+#else
+        std::stringstream str;
+        str << "Trying to convert the negative number " << i << "  to size_t.";
+        OPM_THROW(std::invalid_argument, str.str());
+#endif
     }
 #endif
 
diff --git a/opm/simulators/linalg/gpuistl/device_management.cpp b/opm/simulators/linalg/gpuistl/device_management.cpp
index ee1e202a44..d576ad991d 100644
--- a/opm/simulators/linalg/gpuistl/device_management.cpp
+++ b/opm/simulators/linalg/gpuistl/device_management.cpp
@@ -21,6 +21,9 @@
 
 #include <opm/simulators/flow/FlowGenericVanguard.hpp>
 #include <opm/simulators/utils/DeferredLogger.hpp>
+
+#include <fmt/core.h>
+
 #if HAVE_CUDA
 #include <cuda_runtime.h>
 #include <cuda.h>
diff --git a/tests/gpuistl/test_gpuBlackOilFluidSystem.cu b/tests/gpuistl/test_gpuBlackOilFluidSystem.cu
index 3a99d741ac..c9a80adbd7 100644
--- a/tests/gpuistl/test_gpuBlackOilFluidSystem.cu
+++ b/tests/gpuistl/test_gpuBlackOilFluidSystem.cu
@@ -31,8 +31,6 @@
  * This test requires the presence of opm-parser.
  */
 #include "config.h"
-#include <fmt/format.h>
-#include <iostream>
 
 #include <boost/mpl/list.hpp>
 
@@ -55,8 +53,8 @@
 #include <opm/input/eclipse/Python/Python.hpp>
 #include <opm/input/eclipse/Schedule/Schedule.hpp>
 
-#include <type_traits>
 #include <cmath>
+#include <string>
 
 #include <opm/simulators/linalg/gpuistl/GpuView.hpp>
 #include <opm/simulators/linalg/gpuistl/GpuBuffer.hpp>