diff --git a/opm/simulators/linalg/bda/opencl/BILU0.cpp b/opm/simulators/linalg/bda/opencl/BILU0.cpp index f10d025c8..653847d14 100644 --- a/opm/simulators/linalg/bda/opencl/BILU0.cpp +++ b/opm/simulators/linalg/bda/opencl/BILU0.cpp @@ -165,26 +165,26 @@ bool BILU0::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat this->nnz_jm = jacMat->nnzbs * block_size * block_size; this->nnzbs_jm = jacMat->nnzbs; - int *CSCRowIndices = nullptr; - int *CSCColPointers = nullptr; + std::vector CSCRowIndices; + std::vector CSCColPointers; if (opencl_ilu_reorder == ILUReorder::NONE) { LUmat = std::make_unique(*mat); } else { toOrder.resize(Nb); fromOrder.resize(Nb); - CSCRowIndices = new int[nnzbs_jm]; - CSCColPointers = new int[Nb + 1]; + CSCRowIndices.resize(nnzbs_jm); + CSCColPointers.resize(Nb + 1); rmat = std::make_shared(mat->Nb, mat->nnzbs, block_size); rJacMat = std::make_shared(jacMat->Nb, jacMat->nnzbs, block_size); LUmat = std::make_unique(*rJacMat); - + Timer t_convert; - csrPatternToCsc(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices, CSCColPointers, jacMat->Nb); + csrPatternToCsc(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices.data(), CSCColPointers.data(), jacMat->Nb); if(verbosity >= 3){ - std::ostringstream out; - out << "BILU0 convert CSR to CSC: " << t_convert.stop() << " s"; - OpmLog::info(out.str()); + std::ostringstream out; + out << "BILU0 convert CSR to CSC: " << t_convert.stop() << " s"; + OpmLog::info(out.str()); } } @@ -192,12 +192,12 @@ bool BILU0::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat std::ostringstream out; if (opencl_ilu_reorder == ILUReorder::LEVEL_SCHEDULING) { out << "BILU0 reordering strategy: " << "level_scheduling\n"; - findLevelScheduling(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices, CSCColPointers, jacMat->Nb, &numColors, toOrder.data(), fromOrder.data(), rowsPerColor); + findLevelScheduling(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices.data(), CSCColPointers.data(), jacMat->Nb, &numColors, toOrder.data(), fromOrder.data(), rowsPerColor); for (int iii = 0; iii < numColors; ++iii) { out << "rpc: "<< rowsPerColor[iii] << "\n";} out << "numColors: "<< numColors << "\n"; } else if (opencl_ilu_reorder == ILUReorder::GRAPH_COLORING) { out << "BILU0 reordering strategy: " << "graph_coloring\n"; - findGraphColoring(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices, CSCColPointers, jacMat->Nb, jacMat->Nb, jacMat->Nb, &numColors, toOrder.data(), fromOrder.data(), rowsPerColor); + findGraphColoring(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices.data(), CSCColPointers.data(), jacMat->Nb, jacMat->Nb, jacMat->Nb, &numColors, toOrder.data(), fromOrder.data(), rowsPerColor); for (int iii = 0; iii < numColors; ++iii) { out << "rpc: "<< rowsPerColor[iii] << "\n";} out << "numColors: "<< numColors << "\n"; } else if (opencl_ilu_reorder == ILUReorder::NONE) { @@ -214,17 +214,12 @@ bool BILU0::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat if(verbosity >= 1){ out << "BILU0 analysis took: " << t_analysis.stop() << " s, " << numColors << " colors\n"; } + #if CHOW_PATEL out << "BILU0 CHOW_PATEL: " << CHOW_PATEL << ", CHOW_PATEL_GPU: " << CHOW_PATEL_GPU; #endif OpmLog::info(out.str()); - - if (opencl_ilu_reorder != ILUReorder::NONE) { - delete[] CSCRowIndices; - delete[] CSCColPointers; - } - diagIndex.resize(mat->Nb); invDiagVals.resize(mat->Nb * bs * bs); @@ -400,11 +395,11 @@ bool BILU0::create_preconditioner(BlockedMatrix *mat, BlockedMatrix #if CHOW_PATEL chowPatelIlu.decomposition(queue, context, - LUmat.get(), Lmat.get(), Umat.get(), - invDiagVals, diagIndex, - s.diagIndex, s.invDiagVals, - s.Lvals, s.Lcols, s.Lrows, - s.Uvals, s.Ucols, s.Urows); + LUmat.get(), Lmat.get(), Umat.get(), + invDiagVals, diagIndex, + s.diagIndex, s.invDiagVals, + s.Lvals, s.Lcols, s.Lrows, + s.Uvals, s.Ucols, s.Urows); #else Timer t_copyToGpu; diff --git a/opm/simulators/linalg/bda/opencl/BILU0.hpp b/opm/simulators/linalg/bda/opencl/BILU0.hpp index e2a6b1a52..5fb38e26a 100644 --- a/opm/simulators/linalg/bda/opencl/BILU0.hpp +++ b/opm/simulators/linalg/bda/opencl/BILU0.hpp @@ -133,6 +133,10 @@ public: #endif } + BlockedMatrix* getRJacMat() + { + return rJacMat.get(); + } }; } // namespace Accelerator diff --git a/opm/simulators/linalg/bda/opencl/BISAI.cpp b/opm/simulators/linalg/bda/opencl/BISAI.cpp index 0febeebd4..9bcc934aa 100644 --- a/opm/simulators/linalg/bda/opencl/BISAI.cpp +++ b/opm/simulators/linalg/bda/opencl/BISAI.cpp @@ -91,6 +91,12 @@ bool BISAI::analyze_matrix(BlockedMatrix *mat) return true; } +template +bool BISAI::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat) +{ + return analyze_matrix(mat); +} + template void BISAI::buildLowerSubsystemsStructures(){ lower.subsystemPointers.assign(Nb + 1, 0); @@ -254,6 +260,11 @@ bool BISAI::create_preconditioner(BlockedMatrix *mat) return true; } +template +bool BISAI::create_preconditioner(BlockedMatrix *mat, BlockedMatrix *jacMat) +{ + return create_preconditioner(mat); +} template void BISAI::apply(const cl::Buffer& x, cl::Buffer& y){ diff --git a/opm/simulators/linalg/bda/opencl/BISAI.hpp b/opm/simulators/linalg/bda/opencl/BISAI.hpp index c73b1fd9d..bcaace1b9 100644 --- a/opm/simulators/linalg/bda/opencl/BISAI.hpp +++ b/opm/simulators/linalg/bda/opencl/BISAI.hpp @@ -117,10 +117,13 @@ public: // analysis, find reordering if specified bool analyze_matrix(BlockedMatrix *mat) override; - + + bool analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat) override; // ilu_decomposition bool create_preconditioner(BlockedMatrix *mat) override; + bool create_preconditioner(BlockedMatrix *mat, BlockedMatrix *jacMat) override; + // apply preconditioner, x = prec(y) void apply(const cl::Buffer& y, cl::Buffer& x) override; diff --git a/opm/simulators/linalg/bda/opencl/CPR.cpp b/opm/simulators/linalg/bda/opencl/CPR.cpp index 8de00dab5..047ff1ee0 100644 --- a/opm/simulators/linalg/bda/opencl/CPR.cpp +++ b/opm/simulators/linalg/bda/opencl/CPR.cpp @@ -80,6 +80,10 @@ bool CPR::analyze_matrix(BlockedMatrix *mat_) { return success; } +template +bool CPR::analyze_matrix(BlockedMatrix *mat_, BlockedMatrix *jacMat) { + return analyze_matrix(mat_); +} template bool CPR::create_preconditioner(BlockedMatrix *mat_) { @@ -101,6 +105,10 @@ bool CPR::create_preconditioner(BlockedMatrix *mat_) { return result; } +template +bool CPR::create_preconditioner(BlockedMatrix *mat_, BlockedMatrix *jacMat) { + return create_preconditioner(mat_); +} // return the absolute value of the N elements for which the absolute value is highest double get_absmax(const double *data, const int N) { return std::abs(*std::max_element(data, data + N, [](double a, double b){return std::fabs(a) < std::fabs(b);})); diff --git a/opm/simulators/linalg/bda/opencl/CPR.hpp b/opm/simulators/linalg/bda/opencl/CPR.hpp index 949148dd0..512bc8b4c 100644 --- a/opm/simulators/linalg/bda/opencl/CPR.hpp +++ b/opm/simulators/linalg/bda/opencl/CPR.hpp @@ -126,6 +126,7 @@ public: bool analyze_matrix(BlockedMatrix *mat) override; + bool analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat) override; // set own Opencl variables, but also that of the bilu0 preconditioner void setOpencl(std::shared_ptr& context, std::shared_ptr& queue) override; @@ -135,6 +136,8 @@ public: bool create_preconditioner(BlockedMatrix *mat) override; + bool create_preconditioner(BlockedMatrix *mat, BlockedMatrix *jacMat) override; + int* getToOrder() override { return bilu0->getToOrder();