From 288e548b89ae3fe492a4f3a3339945f564fe0f4c Mon Sep 17 00:00:00 2001 From: Tong Dong Qiu Date: Mon, 15 Nov 2021 10:15:04 +0100 Subject: [PATCH] Changed block_size template to variable for OpenclMatrix --- opm/simulators/linalg/bda/CPR.cpp | 14 +++++++------- opm/simulators/linalg/bda/CPR.hpp | 4 ++-- opm/simulators/linalg/bda/Matrix.cpp | 21 +++------------------ opm/simulators/linalg/bda/Matrix.hpp | 7 ++++--- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/opm/simulators/linalg/bda/CPR.cpp b/opm/simulators/linalg/bda/CPR.cpp index 222826ff5..1d3c2f0eb 100644 --- a/opm/simulators/linalg/bda/CPR.cpp +++ b/opm/simulators/linalg/bda/CPR.cpp @@ -225,21 +225,21 @@ void CPR::create_preconditioner(BlockedMatrix *mat_) { d_Rmatrices.reserve(num_levels - 1); d_invDiags.reserve(num_levels-1); for (Matrix& m : Amatrices) { - d_Amatrices.emplace_back(context.get(), m.N, m.M, m.nnzs); + d_Amatrices.emplace_back(context.get(), m.N, m.M, m.nnzs, 1); } for (Matrix& m : Pmatrices) { - d_Pmatrices.emplace_back(context.get(), m.N, m.M, m.nnzs); + d_Pmatrices.emplace_back(context.get(), m.N, m.M, m.nnzs, 1); d_invDiags.emplace_back(*context, CL_MEM_READ_WRITE, sizeof(double) * m.N); // create a cl::Buffer d_t.emplace_back(*context, CL_MEM_READ_WRITE, sizeof(double) * m.N); } for (Matrix& m : Rmatrices) { - d_Rmatrices.emplace_back(context.get(), m.N, m.M, m.nnzs); + d_Rmatrices.emplace_back(context.get(), m.N, m.M, m.nnzs, 1); d_f.emplace_back(*context, CL_MEM_READ_WRITE, sizeof(double) * m.N); d_u.emplace_back(*context, CL_MEM_READ_WRITE, sizeof(double) * m.N); } d_weights = std::make_unique(*context, CL_MEM_READ_WRITE, sizeof(double) * N); d_rs = std::make_unique(*context, CL_MEM_READ_WRITE, sizeof(double) * N); - d_mat = std::make_unique >(context.get(), Nb, Nb, nnzb); + d_mat = std::make_unique(context.get(), Nb, Nb, nnzb, block_size); d_coarse_y = std::make_unique(*context, CL_MEM_READ_WRITE, sizeof(double) * Nb); d_coarse_x = std::make_unique(*context, CL_MEM_READ_WRITE, sizeof(double) * Nb); }); @@ -419,9 +419,9 @@ void solve_coarse_umfpack(const Matrix *A, std::vector &y, std::vector void CPR::amg_cycle_gpu(const int level, cl::Buffer &y, cl::Buffer &x) { - OpenclMatrix<1> *A = &d_Amatrices[level]; - OpenclMatrix<1> *P = &d_Pmatrices[level]; - OpenclMatrix<1> *R = &d_Rmatrices[level]; + OpenclMatrix *A = &d_Amatrices[level]; + OpenclMatrix *P = &d_Pmatrices[level]; + OpenclMatrix *R = &d_Rmatrices[level]; int Ncur = A->Nb; if (level == num_levels - 1) { diff --git a/opm/simulators/linalg/bda/CPR.hpp b/opm/simulators/linalg/bda/CPR.hpp index a40714307..f17c79ec5 100644 --- a/opm/simulators/linalg/bda/CPR.hpp +++ b/opm/simulators/linalg/bda/CPR.hpp @@ -56,13 +56,13 @@ private: int num_levels; std::vector weights, coarse_vals, coarse_x, coarse_y; std::vector Amatrices, Pmatrices, Rmatrices; // scalar matrices that represent the AMG hierarchy - std::vector > d_Amatrices, d_Pmatrices, d_Rmatrices; // scalar matrices that represent the AMG hierarchy + std::vector d_Amatrices, d_Pmatrices, d_Rmatrices; // scalar matrices that represent the AMG hierarchy std::vector > invDiags; // inverse of diagonal of Amatrices std::vector d_invDiags; std::vector d_t, d_f, d_u; // intermediate vectors used during amg cycle std::unique_ptr d_rs; // use before extracting the pressure std::unique_ptr d_weights; // the quasiimpes weights, used to extract pressure - std::unique_ptr > d_mat; // stores blocked matrix + std::unique_ptr d_mat; // stores blocked matrix std::unique_ptr d_coarse_y, d_coarse_x; // stores the scalar vectors std::once_flag opencl_buffers_allocated; // only allocate OpenCL Buffers once diff --git a/opm/simulators/linalg/bda/Matrix.cpp b/opm/simulators/linalg/bda/Matrix.cpp index fddbf35f3..4278fe3cc 100644 --- a/opm/simulators/linalg/bda/Matrix.cpp +++ b/opm/simulators/linalg/bda/Matrix.cpp @@ -31,8 +31,7 @@ namespace Opm namespace Accelerator { -template -void OpenclMatrix::upload(cl::CommandQueue *queue, double *vals, int *cols, int *rows) { +void OpenclMatrix::upload(cl::CommandQueue *queue, double *vals, int *cols, int *rows) { std::vector events(3); cl_int err = queue->enqueueWriteBuffer(nnzValues, CL_FALSE, 0, sizeof(double) * block_size * block_size * nnzbs, vals, nullptr, &events[0]); @@ -47,13 +46,11 @@ void OpenclMatrix::upload(cl::CommandQueue *queue, double *vals, int } } -template -void OpenclMatrix::upload(cl::CommandQueue *queue, Matrix *matrix) { +void OpenclMatrix::upload(cl::CommandQueue *queue, Matrix *matrix) { upload(queue, matrix->nnzValues.data(), matrix->colIndices.data(), matrix->rowPointers.data()); } -template -void OpenclMatrix::upload(cl::CommandQueue *queue, BlockedMatrix *matrix) { +void OpenclMatrix::upload(cl::CommandQueue *queue, BlockedMatrix *matrix) { upload(queue, matrix->nnzValues, matrix->colIndices, matrix->rowPointers); } @@ -279,18 +276,6 @@ int Matrix::toRDF(int numColors, std::vector& nodesPerColor, } #endif -#define INSTANTIATE_BDA_FUNCTIONS(n) \ -template class OpenclMatrix; - - -INSTANTIATE_BDA_FUNCTIONS(1); -INSTANTIATE_BDA_FUNCTIONS(2); -INSTANTIATE_BDA_FUNCTIONS(3); -INSTANTIATE_BDA_FUNCTIONS(4); -INSTANTIATE_BDA_FUNCTIONS(5); -INSTANTIATE_BDA_FUNCTIONS(6); - -#undef INSTANTIATE_BDA_FUNCTIONS } // namespace Accelerator } // namespace Opm diff --git a/opm/simulators/linalg/bda/Matrix.hpp b/opm/simulators/linalg/bda/Matrix.hpp index 466342f8a..695f07ecc 100644 --- a/opm/simulators/linalg/bda/Matrix.hpp +++ b/opm/simulators/linalg/bda/Matrix.hpp @@ -34,14 +34,14 @@ class Matrix; /// This struct resembles a csr matrix, only doubles are supported /// The matrix data is stored in OpenCL Buffers -template class OpenclMatrix { public: - OpenclMatrix(cl::Context *context, int Nb_, int Mb_, int nnzbs_) + OpenclMatrix(cl::Context *context, int Nb_, int Mb_, int nnzbs_, unsigned int block_size_) : Nb(Nb_), Mb(Mb_), - nnzbs(nnzbs_) + nnzbs(nnzbs_), + block_size(block_size_) { nnzValues = cl::Buffer(*context, CL_MEM_READ_WRITE, sizeof(double) * block_size * block_size * nnzbs); colIndices = cl::Buffer(*context, CL_MEM_READ_WRITE, sizeof(int) * nnzbs); @@ -57,6 +57,7 @@ public: cl::Buffer rowPointers; int Nb, Mb; int nnzbs; + unsigned int block_size; };