subdomain matrix for opencl

Rebased

opm/simulators/linalg/bda/opencl/BILU0.cpp

@@ -165,26 +165,26 @@ bool BILU0<block_size>::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat
     this->nnz_jm = jacMat->nnzbs * block_size * block_size;
     this->nnzbs_jm = jacMat->nnzbs;
 
-    int *CSCRowIndices = nullptr;
+    std::vector<int> CSCRowIndices;
-    int *CSCColPointers = nullptr;
+    std::vector<int> CSCColPointers;
 
     if (opencl_ilu_reorder == ILUReorder::NONE) {
         LUmat = std::make_unique<BlockedMatrix>(*mat);
     } else {
         toOrder.resize(Nb);
         fromOrder.resize(Nb);
-        CSCRowIndices = new int[nnzbs_jm];
+        CSCRowIndices.resize(nnzbs_jm);
-        CSCColPointers = new int[Nb + 1];
+        CSCColPointers.resize(Nb + 1);
         rmat = std::make_shared<BlockedMatrix>(mat->Nb, mat->nnzbs, block_size);
         rJacMat = std::make_shared<BlockedMatrix>(jacMat->Nb, jacMat->nnzbs, block_size);
         LUmat = std::make_unique<BlockedMatrix>(*rJacMat);
             
         Timer t_convert;
-        csrPatternToCsc(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices, CSCColPointers, jacMat->Nb);
+        csrPatternToCsc(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices.data(), CSCColPointers.data(), jacMat->Nb);
         if(verbosity >= 3){
-        std::ostringstream out;
+            std::ostringstream out;
-        out << "BILU0 convert CSR to CSC: " << t_convert.stop() << " s";
+            out << "BILU0 convert CSR to CSC: " << t_convert.stop() << " s";
-        OpmLog::info(out.str());
+            OpmLog::info(out.str());
         }
     }
 
@@ -192,12 +192,12 @@ bool BILU0<block_size>::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat
     std::ostringstream out;
     if (opencl_ilu_reorder == ILUReorder::LEVEL_SCHEDULING) {
         out << "BILU0 reordering strategy: " << "level_scheduling\n";
-        findLevelScheduling(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices, CSCColPointers, jacMat->Nb, &numColors, toOrder.data(), fromOrder.data(), rowsPerColor);
+        findLevelScheduling(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices.data(), CSCColPointers.data(), jacMat->Nb, &numColors, toOrder.data(), fromOrder.data(), rowsPerColor);
         for (int iii = 0; iii < numColors; ++iii) { out << "rpc: "<< rowsPerColor[iii] << "\n";} 
         out << "numColors: "<< numColors << "\n";
     } else if (opencl_ilu_reorder == ILUReorder::GRAPH_COLORING) {
         out << "BILU0 reordering strategy: " << "graph_coloring\n";
-        findGraphColoring<block_size>(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices, CSCColPointers, jacMat->Nb, jacMat->Nb, jacMat->Nb, &numColors, toOrder.data(), fromOrder.data(), rowsPerColor);
+        findGraphColoring<block_size>(jacMat->colIndices, jacMat->rowPointers, CSCRowIndices.data(), CSCColPointers.data(), jacMat->Nb, jacMat->Nb, jacMat->Nb, &numColors, toOrder.data(), fromOrder.data(), rowsPerColor);
         for (int iii = 0; iii < numColors; ++iii) { out << "rpc: "<< rowsPerColor[iii] << "\n";} 
         out << "numColors: "<< numColors << "\n";
     } else if (opencl_ilu_reorder == ILUReorder::NONE) {
@@ -214,17 +214,12 @@ bool BILU0<block_size>::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat
     if(verbosity >= 1){
         out << "BILU0 analysis took: " << t_analysis.stop() << " s, " << numColors << " colors\n";
     }
+
 #if CHOW_PATEL
     out << "BILU0 CHOW_PATEL: " << CHOW_PATEL << ", CHOW_PATEL_GPU: " << CHOW_PATEL_GPU;
 #endif
     OpmLog::info(out.str());
 
-
-    if (opencl_ilu_reorder != ILUReorder::NONE) {
-        delete[] CSCRowIndices;
-        delete[] CSCColPointers;
-    }
-
     diagIndex.resize(mat->Nb);
     invDiagVals.resize(mat->Nb * bs * bs);
 
@@ -400,11 +395,11 @@ bool BILU0<block_size>::create_preconditioner(BlockedMatrix *mat, BlockedMatrix
 
 #if CHOW_PATEL
     chowPatelIlu.decomposition(queue, context,
-        LUmat.get(), Lmat.get(), Umat.get(),
+                               LUmat.get(), Lmat.get(), Umat.get(),
-        invDiagVals, diagIndex,
+                               invDiagVals, diagIndex,
-        s.diagIndex, s.invDiagVals,
+                               s.diagIndex, s.invDiagVals,
-        s.Lvals, s.Lcols, s.Lrows,
+                               s.Lvals, s.Lcols, s.Lrows,
-        s.Uvals, s.Ucols, s.Urows);
+                               s.Uvals, s.Ucols, s.Urows);
 #else
     Timer t_copyToGpu;
 

opm/simulators/linalg/bda/opencl/BILU0.hpp

@@ -133,6 +133,10 @@ public:
 #endif
     }
 
+    BlockedMatrix* getRJacMat()
+    {
+        return rJacMat.get();
+    }
 };
 
 } // namespace Accelerator

opm/simulators/linalg/bda/opencl/BISAI.cpp

@@ -91,6 +91,12 @@ bool BISAI<block_size>::analyze_matrix(BlockedMatrix *mat)
     return true;
 }
 
+template <unsigned int block_size>
+bool BISAI<block_size>::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat)
+{
+    return analyze_matrix(mat);
+}
+
 template <unsigned int block_size>
 void BISAI<block_size>::buildLowerSubsystemsStructures(){
     lower.subsystemPointers.assign(Nb + 1, 0);
@@ -254,6 +260,11 @@ bool BISAI<block_size>::create_preconditioner(BlockedMatrix *mat)
 
     return true;
 }
+template <unsigned int block_size>
+bool BISAI<block_size>::create_preconditioner(BlockedMatrix *mat, BlockedMatrix *jacMat)
+{
+    return create_preconditioner(mat);
+}
 
 template <unsigned int block_size>
 void BISAI<block_size>::apply(const cl::Buffer& x, cl::Buffer& y){

opm/simulators/linalg/bda/opencl/BISAI.hpp

@@ -118,9 +118,12 @@ public:
     // analysis, find reordering if specified
     bool analyze_matrix(BlockedMatrix *mat) override;
     
+    bool analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat) override;
     // ilu_decomposition
     bool create_preconditioner(BlockedMatrix *mat) override;
 
+    bool create_preconditioner(BlockedMatrix *mat, BlockedMatrix *jacMat) override;
+    
     // apply preconditioner, x = prec(y)
     void apply(const cl::Buffer& y, cl::Buffer& x) override;
 

opm/simulators/linalg/bda/opencl/CPR.cpp

@@ -80,6 +80,10 @@ bool CPR<block_size>::analyze_matrix(BlockedMatrix *mat_) {
     return success;
 }
 
+template <unsigned int block_size>
+bool CPR<block_size>::analyze_matrix(BlockedMatrix *mat_, BlockedMatrix *jacMat) {
+    return analyze_matrix(mat_);
+}
 
 template <unsigned int block_size>
 bool CPR<block_size>::create_preconditioner(BlockedMatrix *mat_) {
@@ -101,6 +105,10 @@ bool CPR<block_size>::create_preconditioner(BlockedMatrix *mat_) {
     return result;
 }
 
+template <unsigned int block_size>
+bool CPR<block_size>::create_preconditioner(BlockedMatrix *mat_, BlockedMatrix *jacMat) {
+    return create_preconditioner(mat_);
+}
 // return the absolute value of the N elements for which the absolute value is highest
 double get_absmax(const double *data, const int N) {
     return std::abs(*std::max_element(data, data + N, [](double a, double b){return std::fabs(a) < std::fabs(b);}));

opm/simulators/linalg/bda/opencl/CPR.hpp

@@ -126,6 +126,7 @@ public:
 
     bool analyze_matrix(BlockedMatrix *mat) override;
 
+    bool analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat) override;
     // set own Opencl variables, but also that of the bilu0 preconditioner
     void setOpencl(std::shared_ptr<cl::Context>& context, std::shared_ptr<cl::CommandQueue>& queue) override;
 
@@ -135,6 +136,8 @@ public:
 
     bool create_preconditioner(BlockedMatrix *mat) override;
 
+    bool create_preconditioner(BlockedMatrix *mat, BlockedMatrix *jacMat) override;
+    
     int* getToOrder() override
     {
         return bilu0->getToOrder();