From 3856715bcc4dd60bcf1beb795a45e80e6b66c02e Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Mon, 18 May 2026 09:44:39 +0200 Subject: [PATCH] HypreCpuTransfers: split in two files cpu code in linalg/hypreinterface, gpu code in linalg/gpuistl/hypreinterface --- CMakeLists_files.cmake | 1 + .../linalg/gpuistl/HypreInterface.hpp | 10 +- .../hypreinterface/HypreCpuTransfers.hpp | 173 ++++++---------- .../hypreinterface/HypreCpuTransfers.hpp | 186 ++++++++++++++++++ 4 files changed, 250 insertions(+), 120 deletions(-) create mode 100644 opm/simulators/linalg/hypreinterface/HypreCpuTransfers.hpp diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index 917a3baf27..8010a48c96 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -1421,6 +1421,7 @@ if(HYPRE_FOUND) opm/simulators/linalg/HyprePreconditioner.hpp opm/simulators/linalg/hypreinterface/HypreDataStructures.hpp opm/simulators/linalg/hypreinterface/HypreErrorHandling.hpp + opm/simulators/linalg/hypreinterface/HypreCpuTransfers.hpp opm/simulators/linalg/hypreinterface/HypreUtils.hpp ) endif() diff --git a/opm/simulators/linalg/gpuistl/HypreInterface.hpp b/opm/simulators/linalg/gpuistl/HypreInterface.hpp index 3e51e627df..533a8e9022 100644 --- a/opm/simulators/linalg/gpuistl/HypreInterface.hpp +++ b/opm/simulators/linalg/gpuistl/HypreInterface.hpp @@ -20,7 +20,7 @@ #ifndef OPM_HYPRE_INTERFACE_HPP #define OPM_HYPRE_INTERFACE_HPP -#include +#include #include #include #include @@ -178,7 +178,8 @@ namespace HypreInterface } else #endif { - transferCpuVectorToHypre(vec, hypre_vec, host_arrays, device_arrays, par_info, use_gpu_backend); + linalg::HypreInterface::transferCpuVectorToHypre(vec, hypre_vec, host_arrays, + device_arrays, par_info, use_gpu_backend); } } @@ -196,7 +197,8 @@ namespace HypreInterface } else #endif { - transferHypreToCpuVector(hypre_vec, vec, host_arrays, device_arrays, par_info, use_gpu_backend); + linalg::HypreInterface::transferHypreToCpuVector(hypre_vec, vec, host_arrays, device_arrays, + par_info, use_gpu_backend); } } @@ -215,7 +217,7 @@ namespace HypreInterface } else #endif { - updateMatrixFromCpuMatrix( + linalg::HypreInterface::updateMatrixFromCpuMatrix( matrix, hypre_matrix, sparsity_pattern, host_arrays, device_arrays, use_gpu_backend); } } diff --git a/opm/simulators/linalg/gpuistl/hypreinterface/HypreCpuTransfers.hpp b/opm/simulators/linalg/gpuistl/hypreinterface/HypreCpuTransfers.hpp index 8e13f0aee8..25788beb9f 100644 --- a/opm/simulators/linalg/gpuistl/hypreinterface/HypreCpuTransfers.hpp +++ b/opm/simulators/linalg/gpuistl/hypreinterface/HypreCpuTransfers.hpp @@ -17,8 +17,8 @@ along with OPM. If not, see . */ -#ifndef OPM_HYPRE_CPU_TRANSFERS_HPP -#define OPM_HYPRE_CPU_TRANSFERS_HPP +#ifndef OPM_HYPRE_CPU_TRANSFERS_GPU_HPP +#define OPM_HYPRE_CPU_TRANSFERS_GPU_HPP #include #include @@ -67,56 +67,33 @@ void transferCpuVectorToHypre(const VectorType& cpu_vec, HYPRE_IJVector hypre_vec, linalg::HypreInterface::HypreHostDataArrays& host_arrays, - [[maybe_unused]] const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays, - const linalg::HypreInterface::ParallelInfo& par_info, - bool use_gpu_backend) + const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays, + const linalg::HypreInterface::ParallelInfo& par_info) { const int N = static_cast(host_arrays.indices.size()); using T = typename VectorType::field_type; - if (use_gpu_backend) { -#if HYPRE_USING_CUDA || HYPRE_USING_HIP - // GPU backend with CPU input: use pre-allocated device arrays - if (par_info.owner_first) { - // Direct host-to-device transfer for owner-first ordering - const T* values = &(cpu_vec[0][0]); - hypre_TMemcpy( - device_arrays.vector_buffer_device, values, HYPRE_Real, N, HYPRE_MEMORY_DEVICE, HYPRE_MEMORY_HOST); - OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorSetValues( - hypre_vec, N, device_arrays.indices_device, device_arrays.vector_buffer_device)); - } else { - // Use continuous storage and device buffer for non-owner-first ordering - setContinuousVectorForHypre( - cpu_vec, host_arrays.continuous_vector_values, par_info.local_hypre_to_local_dune); - hypre_TMemcpy(device_arrays.vector_buffer_device, - host_arrays.continuous_vector_values.data(), - HYPRE_Real, - N, - HYPRE_MEMORY_DEVICE, - HYPRE_MEMORY_HOST); - OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorSetValues( - hypre_vec, N, device_arrays.indices_device, device_arrays.vector_buffer_device)); - } -#endif // HYPRE_USING_CUDA || HYPRE_USING_HIP + // GPU backend with CPU input: use pre-allocated device arrays + if (par_info.owner_first) { + // Direct host-to-device transfer for owner-first ordering + const T* values = &(cpu_vec[0][0]); + hypre_TMemcpy( + device_arrays.vector_buffer_device, values, HYPRE_Real, N, HYPRE_MEMORY_DEVICE, HYPRE_MEMORY_HOST); + OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorSetValues( + hypre_vec, N, device_arrays.indices_device, device_arrays.vector_buffer_device)); } else { - // CPU backend with CPU input: use host arrays directly - if (par_info.owner_first) { - // Direct transfer for owner-first ordering - const T* values = &(cpu_vec[0][0]); - OPM_HYPRE_SAFE_CALL( - HYPRE_IJVectorSetValues(hypre_vec, N, const_cast(host_arrays.indices.data()), values)); - } else { - // Use continuous storage for non-owner-first ordering - setContinuousVectorForHypre( - cpu_vec, host_arrays.continuous_vector_values, par_info.local_hypre_to_local_dune); - OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorSetValues(hypre_vec, - N, - const_cast(host_arrays.indices.data()), - host_arrays.continuous_vector_values.data())); - } + // Use continuous storage and device buffer for non-owner-first ordering + setContinuousVectorForHypre( + cpu_vec, host_arrays.continuous_vector_values, par_info.local_hypre_to_local_dune); + hypre_TMemcpy(device_arrays.vector_buffer_device, + host_arrays.continuous_vector_values.data(), + HYPRE_Real, + N, + HYPRE_MEMORY_DEVICE, + HYPRE_MEMORY_HOST); + OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorSetValues( + hypre_vec, N, device_arrays.indices_device, device_arrays.vector_buffer_device)); } - - OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorAssemble(hypre_vec)); } /** @@ -127,53 +104,32 @@ void transferHypreToCpuVector(HYPRE_IJVector hypre_vec, VectorType& cpu_vec, linalg::HypreInterface::HypreHostDataArrays& host_arrays, - [[maybe_unused]] const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays, - const linalg::HypreInterface::ParallelInfo& par_info, - bool use_gpu_backend) + const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays, + const linalg::HypreInterface::ParallelInfo& par_info) { const int N = static_cast(host_arrays.indices.size()); using T = typename VectorType::field_type; - if (use_gpu_backend) { -#if HYPRE_USING_CUDA || HYPRE_USING_HIP - // GPU backend with CPU input: use pre-allocated device arrays - if (par_info.owner_first) { - // Direct device-to-host transfer for owner-first ordering - T* values = &(cpu_vec[0][0]); - OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorGetValues( - hypre_vec, N, device_arrays.indices_device, device_arrays.vector_buffer_device)); - hypre_TMemcpy( - values, device_arrays.vector_buffer_device, HYPRE_Real, N, HYPRE_MEMORY_HOST, HYPRE_MEMORY_DEVICE); - } else { - // Use device buffer and then remap for non-owner-first ordering - OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorGetValues( - hypre_vec, N, device_arrays.indices_device, device_arrays.vector_buffer_device)); - hypre_TMemcpy(host_arrays.continuous_vector_values.data(), - device_arrays.vector_buffer_device, - HYPRE_Real, - N, - HYPRE_MEMORY_HOST, - HYPRE_MEMORY_DEVICE); - setDuneVectorFromContinuousVector( - cpu_vec, host_arrays.continuous_vector_values, par_info.local_hypre_to_local_dune); - } -#endif // HYPRE_USING_CUDA || HYPRE_USING_HIP + // GPU backend with CPU input: use pre-allocated device arrays + if (par_info.owner_first) { + // Direct device-to-host transfer for owner-first ordering + T* values = &(cpu_vec[0][0]); + OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorGetValues( + hypre_vec, N, device_arrays.indices_device, device_arrays.vector_buffer_device)); + hypre_TMemcpy( + values, device_arrays.vector_buffer_device, HYPRE_Real, N, HYPRE_MEMORY_HOST, HYPRE_MEMORY_DEVICE); } else { - // CPU backend with CPU input: use host arrays directly - if (par_info.owner_first) { - // Direct transfer for owner-first ordering - T* values = &(cpu_vec[0][0]); - OPM_HYPRE_SAFE_CALL( - HYPRE_IJVectorGetValues(hypre_vec, N, const_cast(host_arrays.indices.data()), values)); - } else { - // Use continuous storage for non-owner-first ordering - OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorGetValues(hypre_vec, - N, - const_cast(host_arrays.indices.data()), - host_arrays.continuous_vector_values.data())); - setDuneVectorFromContinuousVector( - cpu_vec, host_arrays.continuous_vector_values, par_info.local_hypre_to_local_dune); - } + // Use device buffer and then remap for non-owner-first ordering + OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorGetValues( + hypre_vec, N, device_arrays.indices_device, device_arrays.vector_buffer_device)); + hypre_TMemcpy(host_arrays.continuous_vector_values.data(), + device_arrays.vector_buffer_device, + HYPRE_Real, + N, + HYPRE_MEMORY_HOST, + HYPRE_MEMORY_DEVICE); + setDuneVectorFromContinuousVector( + cpu_vec, host_arrays.continuous_vector_values, par_info.local_hypre_to_local_dune); } } @@ -187,40 +143,25 @@ void updateMatrixFromCpuMatrix(const MatrixType& cpu_matrix, HYPRE_IJMatrix hypre_matrix, const linalg::HypreInterface::SparsityPattern& sparsity_pattern, - const linalg::HypreInterface::HypreHostDataArrays& host_arrays, - [[maybe_unused]] const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays, - bool use_gpu_backend) + const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays) { const auto N = sparsity_pattern.rows.size(); using T = typename MatrixType::field_type; const T* values = &(cpu_matrix[0][0][0][0]); - if (use_gpu_backend) { -#if HYPRE_USING_CUDA || HYPRE_USING_HIP - const auto nnz = cpu_matrix.nonzeroes(); // Total entries including ghost - hypre_TMemcpy( - device_arrays.matrix_buffer_device, values, HYPRE_Real, nnz, HYPRE_MEMORY_DEVICE, HYPRE_MEMORY_HOST); - OPM_HYPRE_SAFE_CALL(HYPRE_IJMatrixSetValues2(hypre_matrix, - N, - device_arrays.ncols_device, - device_arrays.rows_device, - device_arrays.row_indexes_device, - device_arrays.cols_device, - device_arrays.matrix_buffer_device)); -#endif - } else { - OPM_HYPRE_SAFE_CALL(HYPRE_IJMatrixSetValues2(hypre_matrix, - N, - const_cast(sparsity_pattern.ncols.data()), - const_cast(sparsity_pattern.rows.data()), - const_cast(host_arrays.row_indexes.data()), - const_cast(sparsity_pattern.cols.data()), - values)); - } - - OPM_HYPRE_SAFE_CALL(HYPRE_IJMatrixAssemble(hypre_matrix)); + const auto nnz = cpu_matrix.nonzeroes(); // Total entries including ghost + hypre_TMemcpy( + device_arrays.matrix_buffer_device, values, HYPRE_Real, nnz, HYPRE_MEMORY_DEVICE, HYPRE_MEMORY_HOST); + OPM_HYPRE_SAFE_CALL(HYPRE_IJMatrixSetValues2(hypre_matrix, + N, + device_arrays.ncols_device, + device_arrays.rows_device, + device_arrays.row_indexes_device, + device_arrays.cols_device, + device_arrays.matrix_buffer_device)); } + } // namespace Opm::gpuistl::HypreInterface -#endif // OPM_HYPRE_CPU_TRANSFERS_HPP +#endif // OPM_HYPRE_CPU_TRANSFERS_GPU_HPP diff --git a/opm/simulators/linalg/hypreinterface/HypreCpuTransfers.hpp b/opm/simulators/linalg/hypreinterface/HypreCpuTransfers.hpp new file mode 100644 index 0000000000..fa6db41e20 --- /dev/null +++ b/opm/simulators/linalg/hypreinterface/HypreCpuTransfers.hpp @@ -0,0 +1,186 @@ +/* + Copyright 2025 Equinor ASA + + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ + +#ifndef OPM_HYPRE_CPU_TRANSFERS_HPP +#define OPM_HYPRE_CPU_TRANSFERS_HPP + +#include +#include + +#if HYPRE_USING_CUDA +#include +#elif HYPRE_USING_HIP +#include +#endif + +#include +#include <_hypre_utilities.h> + +namespace Opm::linalg::HypreInterface +{ + +/** + * @brief Extract owned vector values in the order expected by HYPRE + */ + template + void + setContinuousVectorForHypre(const VectorType& v, + std::vector& continuous_vector_values, + const std::vector& local_hypre_to_local_dune) + { + // Set v values solution vectors + for (size_t i = 0; i < local_hypre_to_local_dune.size(); ++i) { + continuous_vector_values[i] = v[local_hypre_to_local_dune[i]][0]; + } + } + + /** + * @brief Distribute HYPRE vector values back to original vector positions + */ + template + void + setDuneVectorFromContinuousVector(VectorType& v, + const std::vector& continuous_vector_values, + const std::vector& local_hypre_to_local_dune) + { + // Place values back in original positions + for (size_t i = 0; i < local_hypre_to_local_dune.size(); ++i) { + v[local_hypre_to_local_dune[i]][0] = continuous_vector_values[i]; + } + } + +/** + * @brief Transfer CPU vector to Hypre vector + */ +template +void +transferCpuVectorToHypre(const VectorType& cpu_vec, + HYPRE_IJVector hypre_vec, + linalg::HypreInterface::HypreHostDataArrays& host_arrays, + [[maybe_unused]] const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays, + const linalg::HypreInterface::ParallelInfo& par_info, + bool use_gpu_backend) +{ + const int N = static_cast(host_arrays.indices.size()); + using T = typename VectorType::field_type; + + if (use_gpu_backend) { +#if HYPRE_USING_CUDA || HYPRE_USING_HIP + gpuistl::HypreInterface::transferCpuVectorToHypre(cpu_vec, hypre_vec, host_arrays, + device_arrays, par_info); +#endif // HYPRE_USING_CUDA || HYPRE_USING_HIP + } else { + // CPU backend with CPU input: use host arrays directly + if (par_info.owner_first) { + // Direct transfer for owner-first ordering + const T* values = &(cpu_vec[0][0]); + OPM_HYPRE_SAFE_CALL( + HYPRE_IJVectorSetValues(hypre_vec, N, const_cast(host_arrays.indices.data()), values)); + } else { + // Use continuous storage for non-owner-first ordering + setContinuousVectorForHypre( + cpu_vec, host_arrays.continuous_vector_values, par_info.local_hypre_to_local_dune); + OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorSetValues(hypre_vec, + N, + const_cast(host_arrays.indices.data()), + host_arrays.continuous_vector_values.data())); + } + } + + OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorAssemble(hypre_vec)); +} + +/** + * @brief Transfer Hypre vector to CPU vector + */ +template +void +transferHypreToCpuVector(HYPRE_IJVector hypre_vec, + VectorType& cpu_vec, + linalg::HypreInterface::HypreHostDataArrays& host_arrays, + [[maybe_unused]] const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays, + const linalg::HypreInterface::ParallelInfo& par_info, + bool use_gpu_backend) +{ + const int N = static_cast(host_arrays.indices.size()); + using T = typename VectorType::field_type; + + if (use_gpu_backend) { +#if HYPRE_USING_CUDA || HYPRE_USING_HIP + gpuistl::HypreInterface::transferHypreToCpuVector(hypre_vec, cpu_vec, host_arrays, + device_arrays, par_info); +#endif // HYPRE_USING_CUDA || HYPRE_USING_HIP + } else { + // CPU backend with CPU input: use host arrays directly + if (par_info.owner_first) { + // Direct transfer for owner-first ordering + T* values = &(cpu_vec[0][0]); + OPM_HYPRE_SAFE_CALL( + HYPRE_IJVectorGetValues(hypre_vec, N, const_cast(host_arrays.indices.data()), values)); + } else { + // Use continuous storage for non-owner-first ordering + OPM_HYPRE_SAFE_CALL(HYPRE_IJVectorGetValues(hypre_vec, + N, + const_cast(host_arrays.indices.data()), + host_arrays.continuous_vector_values.data())); + setDuneVectorFromContinuousVector( + cpu_vec, host_arrays.continuous_vector_values, par_info.local_hypre_to_local_dune); + } + } +} + +/** + * @brief Update Hypre matrix from CPU matrix + * Uses HYPRE_IJMatrixSetValues2 with pre-computed row_indexes, which allows us to use the original CPU matrix data + * (with potential ghost values) directly. + */ +template +void +updateMatrixFromCpuMatrix(const MatrixType& cpu_matrix, + HYPRE_IJMatrix hypre_matrix, + const linalg::HypreInterface::SparsityPattern& sparsity_pattern, + const linalg::HypreInterface::HypreHostDataArrays& host_arrays, + [[maybe_unused]] const linalg::HypreInterface::HypreDeviceDataArrays& device_arrays, + bool use_gpu_backend) +{ + const auto N = sparsity_pattern.rows.size(); + + using T = typename MatrixType::field_type; + const T* values = &(cpu_matrix[0][0][0][0]); + + if (use_gpu_backend) { +#if HYPRE_USING_CUDA || HYPRE_USING_HIP + gpuistl::HypreInterface::updateMatrixFromCpuMatrix(cpu_matrix, hypre_matrix, sparsity_pattern, device_arrays); +#endif + } else { + OPM_HYPRE_SAFE_CALL(HYPRE_IJMatrixSetValues2(hypre_matrix, + N, + const_cast(sparsity_pattern.ncols.data()), + const_cast(sparsity_pattern.rows.data()), + const_cast(host_arrays.row_indexes.data()), + const_cast(sparsity_pattern.cols.data()), + values)); + } + + OPM_HYPRE_SAFE_CALL(HYPRE_IJMatrixAssemble(hypre_matrix)); +} + +} // namespace Opm::linalg::HypreInterface + +#endif // OPM_HYPRE_CPU_TRANSFERS_HPP