reenable normal opencl

Rebased

ebos/eclcpgridvanguard.hh

@@ -192,7 +192,6 @@ protected:
     }
 
     std::unique_ptr<TransmissibilityType> globalTrans_;
-    //std::vector<int> cell_part_;
 };
 
 } // namespace Opm

ebos/eclgenericcpgridvanguard.cc

@@ -181,6 +181,7 @@ void EclGenericCpGridVanguard<ElementMapper,GridView,Scalar>::doLoadBalance_(Dun
         // cells that exist only on other ranks even in the case of distributed wells
         // But we need all connections to figure out the first cell of a well (e.g. for
         // pressure). Hence this is now skipped. Rank 0 had everything even before.
+
         if (numJacobiBlocks > 0 && mpiSize == 1) {
             const auto wells = schedule.getWellsatEnd();
             cell_part_.resize(grid_->numCells());

opm/simulators/linalg/bda/BdaBridge.cpp

@@ -211,11 +211,6 @@ void BdaBridge<BridgeMatrix, BridgeVector, block_size>::solve_system([[maybe_unu
         const int nnzb = (h_rows.empty()) ? mat->nonzeroes() : h_rows.back();
         const int nnz = nnzb * dim * dim;
 
-        static std::vector<int> bm_h_rows;
-        static std::vector<int> bm_h_cols;
-        const int bm_nnzb = (bm_h_rows.empty()) ? blockMat->nonzeroes() : bm_h_rows.back();
-        const int bm_nnz = bm_nnzb * dim * dim;
-        
         if (dim != 3) {
             OpmLog::warning("BdaSolver only accepts blocksize = 3 at this time, will use Dune for the remainder of the program");
             use_gpu = use_fpga = false;
@@ -228,15 +223,9 @@ void BdaBridge<BridgeMatrix, BridgeVector, block_size>::solve_system([[maybe_unu
             copySparsityPatternFromISTL(*mat, h_rows, h_cols);
         }
 
-        if (bm_h_rows.capacity() == 0) {
-            bm_h_rows.reserve(Nb+1);
-            bm_h_cols.reserve(bm_nnzb);
-            copySparsityPatternFromISTL(*blockMat, bm_h_rows, bm_h_cols);
-        }
-
         Dune::Timer t_zeros;
         int numZeros = checkZeroDiagonal(*mat);
-        int bm_numZeros = checkZeroDiagonal(*blockMat);
+
         if (verbosity >= 2) {
             std::ostringstream out;
             out << "Checking zeros took: " << t_zeros.stop() << " s, found " << numZeros << " zeros";
@@ -247,16 +236,34 @@ void BdaBridge<BridgeMatrix, BridgeVector, block_size>::solve_system([[maybe_unu
         /////////////////////////
         // actually solve
         SolverStatus status; 
-        if (numJacobiBlocks > 1)
+        if (numJacobiBlocks < 2) {
+            // assume that underlying data (nonzeroes) from mat (Dune::BCRSMatrix) are contiguous, if this is not the case, the chosen BdaSolver is expected to perform undefined behaviour
+            status = backend->solve_system(N, nnz, dim, static_cast<double*>(&(((*mat)[0][0][0][0]))), h_rows.data(), h_cols.data(), static_cast<double*>(&(b[0][0])), wellContribs, result);
+        }
+        else {
+            static std::vector<int> bm_h_rows;
+            static std::vector<int> bm_h_cols;
+            const int bm_nnzb = (bm_h_rows.empty()) ? blockMat->nonzeroes() : bm_h_rows.back();
+            const int bm_nnz = bm_nnzb * dim * dim;
+            
+            if (bm_h_rows.capacity() == 0) {
+                bm_h_rows.reserve(Nb+1);
+                bm_h_cols.reserve(bm_nnzb);
+                copySparsityPatternFromISTL(*blockMat, bm_h_rows, bm_h_cols);
+            }
+
+            int bm_numZeros = checkZeroDiagonal(*blockMat);
+            if (verbosity >= 2) {
+                std::ostringstream out;
+                out << "Checking zeros took: " << t_zeros.stop() << " s, found " << numZeros << " zeros";
+                OpmLog::info(out.str());
+            }
+            
             status = backend->solve_system2(N, nnz, dim, static_cast<double*>(&(((*mat)[0][0][0][0]))), h_rows.data(), h_cols.data(), static_cast<double*>(&(b[0][0])),
                                             bm_nnz, static_cast<double*>(&(((*blockMat)[0][0][0][0]))), bm_h_rows.data(), bm_h_cols.data(),
                                             wellContribs, result);
-        else
+        }
-            status = backend->solve_system(N, nnz, dim, static_cast<double*>(&(((*mat)[0][0][0][0]))), h_rows.data(), h_cols.data(), static_cast<double*>(&(b[0][0])), wellContribs, result);
 
-            
-        // assume that underlying data (nonzeroes) from mat (Dune::BCRSMatrix) are contiguous, if this is not the case, the chosen BdaSolver is expected to perform undefined behaviour
-        //SolverStatus status = backend->solve_system(N, nnz, dim, static_cast<double*>(&(((*mat)[0][0][0][0]))), h_rows.data(), h_cols.data(), static_cast<double*>(&(b[0][0])), wellContribs, result);
         switch(status) {
         case SolverStatus::BDA_SOLVER_SUCCESS:
             //OpmLog::info("BdaSolver converged");

opm/simulators/linalg/bda/FPGASolverBackend.cpp

@@ -223,6 +223,7 @@ void FpgaSolverBackend<block_size>::get_result(double *x_)
     }
 } // end get_result()
 
+
 template <unsigned int block_size>
 SolverStatus FpgaSolverBackend<block_size>::solve_system(int N_, int nnz_, int dim, double *vals, int *rows, int *cols, double *b, WellContributions&, BdaResult &res)
 {
@@ -255,7 +256,8 @@ SolverStatus FpgaSolverBackend<block_size>::solve_system2(int N_, int nnz_, int
 					   int nnz2, double *vals2, int *rows2, int *cols2,
 					   WellContributions& wellContribs, BdaResult &res)
 {
-    return SolverStatus::BDA_SOLVER_ANALYSIS_FAILED;
+    (void) nnz2; (void) vals2; (void) rows2; (void) cols2;
+    return solve_system(N_, nnz_, dim, vals, rows, cols, b, wellContribs, res);
 }
 
 template <unsigned int block_size>

opm/simulators/linalg/bda/amgclSolverBackend.cpp

@@ -365,7 +365,8 @@ SolverStatus amgclSolverBackend<block_size>::solve_system2(int N_, int nnz_, int
 							   int nnz2, double *vals2, int *rows2, int *cols2,
 							   WellContributions& wellContribs, BdaResult &res)
 {
-    return SolverStatus::BDA_SOLVER_ANALYSIS_FAILED;
+    (void) nnz2; (void) vals2; (void) rows2; (void) cols2;
+    return solve_system(N_, nnz_, dim, vals, rows, cols, b, wellContribs, res);
 }
     
 template <unsigned int block_size>
@@ -380,7 +381,6 @@ SolverStatus amgclSolverBackend<block_size>::solve_system(int N_, int nnz_, int
 }
 
 
-
 #define INSTANTIATE_BDA_FUNCTIONS(n)                                                                \
 template amgclSolverBackend<n>::amgclSolverBackend(int, int, double, unsigned int, unsigned int);   \
 

opm/simulators/linalg/bda/cuda/cusparseSolverBackend.cu

@@ -506,7 +506,8 @@ SolverStatus cusparseSolverBackend<block_size>::solve_system2(int N_, int nnz_,
 					   int nnz2, double *vals2, int *rows2, int *cols2,
 					   WellContributions& wellContribs, BdaResult &res)
 {
-    return SolverStatus::BDA_SOLVER_ANALYSIS_FAILED;
+    (void) nnz2; (void) vals2; (void) rows2; (void) cols2;
+    return solve_system(N_, nnz_, dim, vals, rows, cols, b, wellContribs, res);
 }
 
 #define INSTANTIATE_BDA_FUNCTIONS(n)                                                       \

opm/simulators/linalg/bda/opencl/BILU0.cpp

@@ -365,11 +365,11 @@ template <unsigned int block_size>
 bool BILU0<block_size>::create_preconditioner(BlockedMatrix *mat, BlockedMatrix *jacMat)
 {
     const unsigned int bs = block_size;
-    auto *m = mat;
+
     auto *jm = jacMat;
 
     if (opencl_ilu_reorder != ILUReorder::NONE) {
-        m = rmat.get();
+
         jm = rJacMat.get();
         Timer t_reorder;
         reorderBlockedMatrixByPattern(mat, toOrder.data(), fromOrder.data(), rmat.get());

opm/simulators/linalg/bda/opencl/BISAI.cpp

@@ -94,6 +94,7 @@ bool BISAI<block_size>::analyze_matrix(BlockedMatrix *mat)
 template <unsigned int block_size>
 bool BISAI<block_size>::analyze_matrix(BlockedMatrix *mat, BlockedMatrix *jacMat)
 {
+    (void) jacMat;
     return analyze_matrix(mat);
 }
 
@@ -263,6 +264,7 @@ bool BISAI<block_size>::create_preconditioner(BlockedMatrix *mat)
 template <unsigned int block_size>
 bool BISAI<block_size>::create_preconditioner(BlockedMatrix *mat, BlockedMatrix *jacMat)
 {
+    (void) jacMat;
     return create_preconditioner(mat);
 }
 

opm/simulators/linalg/bda/opencl/CPR.cpp

@@ -82,6 +82,7 @@ bool CPR<block_size>::analyze_matrix(BlockedMatrix *mat_) {
 
 template <unsigned int block_size>
 bool CPR<block_size>::analyze_matrix(BlockedMatrix *mat_, BlockedMatrix *jacMat) {
+    (void) jacMat;
     return analyze_matrix(mat_);
 }
 
@@ -107,6 +108,7 @@ bool CPR<block_size>::create_preconditioner(BlockedMatrix *mat_) {
 
 template <unsigned int block_size>
 bool CPR<block_size>::create_preconditioner(BlockedMatrix *mat_, BlockedMatrix *jacMat) {
+    (void) jacMat;
     return create_preconditioner(mat_);
 }
 // return the absolute value of the N elements for which the absolute value is highest

opm/simulators/linalg/bda/opencl/openclSolverBackend.cpp

@@ -590,7 +590,11 @@ template <unsigned int block_size>
 bool openclSolverBackend<block_size>::analyze_matrix() {
     Timer t;
 
-    bool success = prec->analyze_matrix(mat.get(), jacMat.get());
+    bool success;
+    if (blockJacVersion)
+        success = prec->analyze_matrix(mat.get(), jacMat.get());
+    else
+        success = prec->analyze_matrix(mat.get());
 
     if (opencl_ilu_reorder == ILUReorder::NONE) {
         rmat = mat.get();
@@ -641,7 +645,11 @@ template <unsigned int block_size>
 bool openclSolverBackend<block_size>::create_preconditioner() {
     Timer t;
 
-    bool result = prec->create_preconditioner(mat.get(), jacMat.get());
+    bool result;
+    if (blockJacVersion)
+        result = prec->create_preconditioner(mat.get(), jacMat.get());
+    else
+        result = prec->create_preconditioner(mat.get());
 
     if (verbosity > 2) {
         std::ostringstream out;
@@ -730,6 +738,7 @@ SolverStatus openclSolverBackend<block_size>::solve_system2(int N_, int nnz_, in
                            WellContributions& wellContribs, BdaResult &res)
 {
     if (initialized == false) {
+        blockJacVersion = true;
         initialize2(N_, nnz_,  dim, vals, rows, cols, nnz2, vals2, rows2, cols2);
         if (analysis_done == false) {
             if (!analyze_matrix()) {

opm/simulators/linalg/bda/opencl/openclSolverBackend.hpp

@@ -65,6 +65,8 @@ private:
 
     int jac_nnz;
     int jac_nnzb;
+    bool blockJacVersion = false;
+
     std::unique_ptr<Preconditioner<block_size> > prec;
                                                                   // can perform blocked ILU0 and AMG on pressure component
     bool is_root;                                                 // allow for nested solvers, the root solver is called by BdaBridge