From 88db0c3b3eadd33b7b65a3e39589ad27828cb891 Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Thu, 12 Feb 2026 09:41:40 +0100 Subject: [PATCH 1/4] move setting of test properties into tests hook --- CMakeLists.txt | 157 ++++++++++++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 67 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e2ae47c8bf..bcfddbe035 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -197,6 +197,96 @@ macro(opm-simulators_tests_hook) if(OPM_ENABLE_PYTHON) include(${CMAKE_CURRENT_SOURCE_DIR}/pythonIntegrationTests.cmake) endif() + + if(CUDA_FOUND) + if(USE_GPU_BRIDGE) + set_tests_properties(cusparseSolver PROPERTIES LABELS gpu_cuda) + endif() + + # CUISTL + set(gpu_label "gpu_cuda") + if(USE_HIP) + set(gpu_label "gpu_hip") + endif() + + set_tests_properties( + cusparse_safe_call + cublas_safe_call + gpu_safe_call + cuda_check_last_error + cublas_handle + GpuJac + GpuDILU + cusparse_handle + cuSparse_matrix_operations + cuVector_operations + GpuVector + GpuSparseMatrix + GpuSeqILU0 + solver_adapter + GpuBuffer + GpuView + gpu_ad + gpu_linear_two_phase_material + gpuPvt + gpu_resources + gpu_smart_pointers + is_gpu_pointer + throw_macros_on_gpu + preconditioner_factory_gpu + gpuBlackOilFluidSystem + blackoilfluidstategpu + conditional_storage + GpuPressureTransferPolicy + deviceBlockOperations + primary_variables_gpu + MiniVector + MiniMatrix + GpuSparseTable + PROPERTIES + LABELS + ${gpu_label} + ) + if(MPI_FOUND) + set_tests_properties(GpuOwnerOverlapCopy PROPERTIES LABELS ${gpu_label}) + endif() + endif() + + if(HYPRE_FOUND) + if (HYPRE_USING_CUDA) + set_tests_properties(HyprePreconditionerGPU PROPERTIES LABELS gpu_cuda) + set_tests_properties(HypreInterfaceGPU PROPERTIES LABELS gpu_cuda) + elseif (HYPRE_USING_HIP) + set_tests_properties(HyprePreconditionerGPU PROPERTIES LABELS gpu_hip) + set_tests_properties(HypreInterfaceGPU PROPERTIES LABELS gpu_hip) + endif() + endif() + + if(AMGX_FOUND) + set_tests_properties(AmgxInterface PROPERTIES LABELS gpu_cuda) + set_tests_properties(AmgxPreconditioner PROPERTIES LABELS gpu_cuda) + endif() + + if(USE_GPU_BRIDGE) + if(OpenCL_FOUND) + set_tests_properties( + openclSolver + solvetransposed3x3 + csrToCscOffsetMap + PROPERTIES + LABELS + gpu_opencl + ) + endif() + + if(ROCALUTION_FOUND) + set_tests_properties(rocalutionSolver PROPERTIES LABELS gpu_rocm) + endif() + + if(rocsparse_FOUND AND rocblas_FOUND) + set_tests_properties(rocsparseSolver PROPERTIES LABELS gpu_rocm) + endif() + endif() endmacro() macro(opm-simulators_targets_hook) @@ -764,85 +854,18 @@ if(CUDA_FOUND) target_link_libraries(${tgt} PRIVATE CUDA::cudart) endforeach() endif() - if(USE_GPU_BRIDGE) - set_tests_properties(cusparseSolver PROPERTIES LABELS gpu_cuda) - endif() - - # CUISTL - set(gpu_label "gpu_cuda") - if(USE_HIP) - set(gpu_label "gpu_hip") - endif() - - set_tests_properties(cusparse_safe_call - cublas_safe_call - gpu_safe_call - cuda_check_last_error - cublas_handle - GpuJac - GpuDILU - cusparse_handle - cuSparse_matrix_operations - cuVector_operations - GpuVector - GpuSparseMatrix - GpuSeqILU0 - solver_adapter - GpuBuffer - GpuView - gpu_ad - gpu_linear_two_phase_material - gpuPvt - gpu_resources - gpu_smart_pointers - is_gpu_pointer - throw_macros_on_gpu - preconditioner_factory_gpu - gpuBlackOilFluidSystem - blackoilfluidstategpu - conditional_storage - GpuPressureTransferPolicy - deviceBlockOperations - primary_variables_gpu - MiniVector - MiniMatrix - GpuSparseTable - PROPERTIES LABELS ${gpu_label}) - if(MPI_FOUND) - set_tests_properties(GpuOwnerOverlapCopy PROPERTIES LABELS ${gpu_label}) - endif() -endif() - -if(HYPRE_FOUND) - if (HYPRE_USING_CUDA) - set_tests_properties(HyprePreconditionerGPU PROPERTIES LABELS gpu_cuda) - set_tests_properties(HypreInterfaceGPU PROPERTIES LABELS gpu_cuda) - elseif (HYPRE_USING_HIP) - set_tests_properties(HyprePreconditionerGPU PROPERTIES LABELS gpu_hip) - set_tests_properties(HypreInterfaceGPU PROPERTIES LABELS gpu_hip) - endif() -endif() - -if(AMGX_FOUND) - set_tests_properties(AmgxInterface PROPERTIES LABELS gpu_cuda) - set_tests_properties(AmgxPreconditioner PROPERTIES LABELS gpu_cuda) endif() if(USE_GPU_BRIDGE) if(OpenCL_FOUND) target_link_libraries( opmsimulators PUBLIC ${OpenCL_LIBRARIES} ) - set_tests_properties(openclSolver solvetransposed3x3 csrToCscOffsetMap - PROPERTIES LABELS gpu_opencl) endif() if(ROCALUTION_FOUND) - target_include_directories(opmsimulators PUBLIC ${rocalution_INCLUDE_DIR}/rocalution) set_tests_properties(rocalutionSolver PROPERTIES LABELS gpu_rocm) endif() if(rocsparse_FOUND AND rocblas_FOUND) - target_link_libraries( opmsimulators PUBLIC roc::rocsparse ) - target_link_libraries( opmsimulators PUBLIC roc::rocblas ) set_tests_properties(rocsparseSolver PROPERTIES LABELS gpu_rocm) endif() From 2a37b5827793fff94852af9e4045d536de667b4a Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Wed, 11 Feb 2026 14:18:11 +0100 Subject: [PATCH 2/4] hip: avoid setting CUDA_FOUND to ON CUDA was not found. let's use the appropriate variables --- CMakeLists.txt | 3 +-- CMakeLists_files.cmake | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bcfddbe035..e8e72f7bfb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,7 +198,7 @@ macro(opm-simulators_tests_hook) include(${CMAKE_CURRENT_SOURCE_DIR}/pythonIntegrationTests.cmake) endif() - if(CUDA_FOUND) + if(CUDA_FOUND OR hip_FOUND) if(USE_GPU_BRIDGE) set_tests_properties(cusparseSolver PROPERTIES LABELS gpu_cuda) endif() @@ -382,7 +382,6 @@ if(CONVERT_CUDA_TO_HIP) enable_language(HIP) message("CUDA code will be hipified") set(HAVE_CUDA 1) # we still need this defined so that the preprocessor does not remove the code - set(CUDA_FOUND ON) set(USE_HIP 1) find_package(hip REQUIRED) find_package(hipsparse REQUIRED) diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index 120ea9063a..0b6cec684d 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -30,7 +30,7 @@ macro (ADD_CUDA_OR_HIP_FILE LIST DIR FILE) set (cuda_file_path "${PROJECT_SOURCE_DIR}/${DIR}/gpuistl/${FILE}") - if(CUDA_FOUND AND NOT CONVERT_CUDA_TO_HIP) + if(CUDA_FOUND) list (APPEND ${LIST} "${DIR}/gpuistl/${FILE}") else() # we must hipify the code @@ -308,7 +308,7 @@ if (Damaris_FOUND AND MPI_FOUND AND USE_DAMARIS_LIB) endif() # add these files if we should compile the hip code -if (HAVE_CUDA) +if(CUDA_FOUND OR hip_FOUND) list(APPEND MAIN_SOURCE_FILES opm/simulators/linalg/gpuistl/device_management.hpp) # should not be hipified to make main independant of library ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg device_management.cpp) ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg detail/CuBlasHandle.cpp) @@ -544,7 +544,7 @@ if(CUDA_FOUND) endif() endif() -if (HAVE_CUDA) +if(CUDA_FOUND OR hip_FOUND) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_converttofloatadapter.cpp) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cublas_handle.cpp) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cublas_safe_call.cpp) From 89777c8dffd73ae002087d154ce0407619e94832 Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Thu, 12 Feb 2026 08:57:13 +0100 Subject: [PATCH 3/4] add convenience wrapper function --- CMakeLists.txt | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e8e72f7bfb..a04e9aba89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,28 @@ option(USE_HYPRE "Use the Hypre library for linear solvers?" OFF) set(OPM_COMPILE_COMPONENTS "2;3;4;5;6;7" CACHE STRING "The components to compile support for") option(USE_OPENCL "Enable OpenCL support?" ON) +# Wrapper for opm_add_target_options that also adds some (optional) +# common links and dependencies +function(simulators_add_target_options) + cmake_parse_arguments(PARAM "" "TARGET" "" ${ARGN}) + if(NOT PARAM_TARGET) + message(FATAL_ERROR "Function needs a TARGET parameter") + endif() + + opm_add_target_options(TARGET ${PARAM_TARGET}) + if(TARGET fmt::fmt) + target_link_libraries(${PARAM_TARGET} PRIVATE fmt::fmt) + endif() + + if(USE_TRACY_PROFILER AND Tracy_FOUND) + target_link_libraries(${PARAM_TARGET} PRIVATE Tracy::TracyClient) + endif() + + if(TARGET opmcommon) + add_dependencies(${PARAM_TARGET} opmcommon) + endif() +endfunction() + macro(opm-simulators_prereqs_hook) if(MPI_FOUND AND HDF5_FOUND AND NOT HDF5_IS_PARALLEL) message( @@ -645,7 +667,7 @@ set_property(TARGET moduleVersion PROPERTY POSITION_INDEPENDENT_CODE ON) opm_add_target_options(TARGET moduleVersion) add_library(MainDispatchDynamic OBJECT opm/simulators/flow/MainDispatchDynamic.cpp) -opm_add_target_options(TARGET MainDispatchDynamic) +simulators_add_target_options(TARGET MainDispatchDynamic) set_property(TARGET MainDispatchDynamic PROPERTY POSITION_INDEPENDENT_CODE ON) # We only wait for the hipification process if we are compiling HIP. @@ -656,18 +678,6 @@ if (CONVERT_CUDA_TO_HIP) add_dependencies(MainDispatchDynamic hipified_headers) endif() -if(TARGET fmt::fmt) - target_link_libraries(MainDispatchDynamic PRIVATE fmt::fmt) -endif() - -if(TARGET opmcommon) - add_dependencies(MainDispatchDynamic opmcommon) -endif() - -if(USE_TRACY_PROFILER AND Tracy_FOUND) - target_link_libraries(MainDispatchDynamic PRIVATE Tracy::TracyClient) -endif() - # Strictly we only depend on the update-version target, # but this is not exposed in a super-build. add_dependencies(moduleVersion opmsimulators) @@ -685,17 +695,8 @@ set(FLOW_VARIANT_MODELS brine_energy onephase onephase_energy) set(FLOW_TGTS) foreach(OBJ ${COMMON_MODELS} ${FLOW_MODELS} ${FLOW_VARIANT_MODELS}) add_library(flow_lib${OBJ} OBJECT flow/flow_${OBJ}.cpp) - opm_add_target_options(TARGET flow_lib${OBJ}) + simulators_add_target_options(TARGET flow_lib${OBJ}) list(APPEND FLOW_TGTS $) - if(TARGET fmt::fmt) - target_link_libraries(flow_lib${OBJ} PRIVATE fmt::fmt) - endif() - if(TARGET opmcommon) - add_dependencies(flow_lib${OBJ} opmcommon) - endif() - if(USE_TRACY_PROFILER AND Tracy_FOUND) - target_link_libraries(flow_lib${OBJ} PRIVATE Tracy::TracyClient) - endif() # We only wait for the hipification process if we are compiling HIP. # That is, we need to wait for the HIP header files to be generated From d5f1ac9937ec3e5ae98ff977de3863f0374a0d10 Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Wed, 11 Feb 2026 14:25:03 +0100 Subject: [PATCH 4/4] move hip setup to prereqs hook move config variables to the config hook and linking to the targets hook --- CMakeLists.txt | 79 +++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a04e9aba89..70978820c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,15 @@ function(simulators_add_target_options) if(TARGET opmcommon) add_dependencies(${PARAM_TARGET} opmcommon) endif() + + # We only wait for the hipification process if we are compiling HIP. + # That is, we need to wait for the HIP header files to be generated + # before the rest of the code can compile. HIP header file includes + # anything in gpuistl, say. + if (CONVERT_CUDA_TO_HIP) + add_dependencies(${PARAM_TARGET} hipified_headers) + target_link_libraries(${PARAM_TARGET} PRIVATE roc::hipblas roc::hipsparse) + endif() endfunction() macro(opm-simulators_prereqs_hook) @@ -96,6 +105,27 @@ macro(opm-simulators_prereqs_hook) include(DownloadPyBind11) endif() endif() + + if(CONVERT_CUDA_TO_HIP) + message("CUDA code will be hipified") + enable_language(HIP) + find_package(hip REQUIRED) + find_package(hipsparse REQUIRED) + find_package(hipblas REQUIRED) + find_program(HIPIFY_PERL_COMMAND + NAMES hipify-perl + HINTS + PATH /opt/rocm/bin + ) + if(HIPIFY_PERL_COMMAND) + add_executable(hipify-perl IMPORTED GLOBAL) + set_target_properties(hipify-perl + PROPERTIES IMPORTED_LOCATION ${HIPIFY_PERL_COMMAND} + ) + else() + message(FATAL_ERROR "hipify-perl, required to convert CUDA to HIP, not found") + endif() + endif() endmacro() macro(opm-simulators_sources_hook) @@ -199,6 +229,11 @@ macro(opm-simulators_config_hook) else() unset(HAVE_DAMARIS) # Required until we no longer use find_package_deps() endif() + + if(CONVERT_CUDA_TO_HIP) + set(HAVE_CUDA 1) # we still need this defined so that the preprocessor does not remove the code + set(USE_HIP 1) + endif() endmacro() macro(opm-simulators_tests_hook) @@ -369,6 +404,10 @@ macro(opm-simulators_targets_hook) if (Damaris_FOUND AND MPI_FOUND AND USE_DAMARIS_LIB) target_link_libraries(opmsimulators PUBLIC damaris) endif() + + if(CONVERT_CUDA_TO_HIP) + target_link_libraries(opmsimulators PUBLIC roc::hipblas roc::hipsparse) + endif() endmacro() macro(opm-simulators_install_hook) @@ -400,30 +439,6 @@ list(REMOVE_DUPLICATES OPM_COMPILE_COMPONENTS) # this will be used in a template argument pack. string(REPLACE ";" "," OPM_COMPILE_COMPONENTS_TEMPLATE_LIST "${OPM_COMPILE_COMPONENTS}") -if(CONVERT_CUDA_TO_HIP) - enable_language(HIP) - message("CUDA code will be hipified") - set(HAVE_CUDA 1) # we still need this defined so that the preprocessor does not remove the code - set(USE_HIP 1) - find_package(hip REQUIRED) - find_package(hipsparse REQUIRED) - find_package(hipblas REQUIRED) - link_libraries(roc::hipblas roc::hipsparse) - find_program(HIPIFY_PERL_COMMAND - NAMES hipify-perl - HINTS - PATH /opt/rocm/bin - ) - if(HIPIFY_PERL_COMMAND) - add_executable(hipify-perl IMPORTED GLOBAL) - set_target_properties(hipify-perl - PROPERTIES IMPORTED_LOCATION ${HIPIFY_PERL_COMMAND} - ) - else() - message(FATAL_ERROR "hipify-perl required to convert CUDA to HIP not found") - endif() -endif() - if(SIBLING_SEARCH AND NOT opm-common_DIR) # guess the sibling dir get_filename_component(_leaf_dir_name ${PROJECT_BINARY_DIR} NAME) @@ -670,14 +685,6 @@ add_library(MainDispatchDynamic OBJECT opm/simulators/flow/MainDispatchDynamic.c simulators_add_target_options(TARGET MainDispatchDynamic) set_property(TARGET MainDispatchDynamic PROPERTY POSITION_INDEPENDENT_CODE ON) -# We only wait for the hipification process if we are compiling HIP. -# That is, we need to wait for the HIP header files to be generated -# before the rest of the code can compile. HIP header file includes -# anything in gpuistl, say. -if (CONVERT_CUDA_TO_HIP) - add_dependencies(MainDispatchDynamic hipified_headers) -endif() - # Strictly we only depend on the update-version target, # but this is not exposed in a super-build. add_dependencies(moduleVersion opmsimulators) @@ -698,14 +705,6 @@ foreach(OBJ ${COMMON_MODELS} ${FLOW_MODELS} ${FLOW_VARIANT_MODELS}) simulators_add_target_options(TARGET flow_lib${OBJ}) list(APPEND FLOW_TGTS $) - # We only wait for the hipification process if we are compiling HIP. - # That is, we need to wait for the HIP header files to be generated - # before the rest of the code can compile. HIP header file includes - # anything in gpuistl, say. - if(CONVERT_CUDA_TO_HIP) - add_dependencies(flow_lib${OBJ} hipified_headers) - endif() - opm_add_test(flow_${OBJ} ONLY_COMPILE SOURCES