From e6dff88eb7ce850325f721a1db2979128be12fbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?B=C3=A5rd=20Skaflestad?= <Bard.Skaflestad@sintef.no>
Date: Tue, 31 May 2022 17:55:50 +0200
Subject: [PATCH] Temporarily Limit Block-Jacobi Partitioner to OpenCL Only

The block-Jacobi partitioner (commit e360c00b7) uses grid interfaces
that are only available in CpGrid and this blocks introducing other
grid managers such as ALUGrid.  Since the partitioner was only added
for OpenCL, guard special purposes accesses with HAVE_OPENCL.

This is a temporary measure and we will venture to restore the
partitioner later, although possibly restricted to CpGrid only.
---
 ebos/eclbasevanguard.hh                  | 16 ++++++++++++++++
 ebos/eclgenericcpgridvanguard.cc         |  2 ++
 ebos/eclgenericvanguard.hh               | 14 ++++++++++++--
 opm/simulators/linalg/ISTLSolverEbos.hpp |  7 ++++++-
 4 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/ebos/eclbasevanguard.hh b/ebos/eclbasevanguard.hh
index 30a3c1aa8..60a9e4716 100644
--- a/ebos/eclbasevanguard.hh
+++ b/ebos/eclbasevanguard.hh
@@ -84,10 +84,14 @@ template<class TypeTag, class MyTypeTag>
 struct EdgeWeightsMethod {
     using type = UndefinedProperty;
 };
+
+#if HAVE_OPENCL
 template<class TypeTag, class MyTypeTag>
 struct NumJacobiBlocks {
     using type = UndefinedProperty;
 };
+#endif  // HAVE_OPENCL
+
 template<class TypeTag, class MyTypeTag>
 struct OwnerCellsFirst {
     using type = UndefinedProperty;
@@ -136,10 +140,14 @@ template<class TypeTag>
 struct EdgeWeightsMethod<TypeTag, TTag::EclBaseVanguard> {
     static constexpr int value = 1;
 };
+
+#if HAVE_OPENCL
 template<class TypeTag>
 struct NumJacobiBlocks<TypeTag, TTag::EclBaseVanguard> {
     static constexpr int value = 0;
 };
+#endif // HAVE_OPENCL
+
 template<class TypeTag>
 struct OwnerCellsFirst<TypeTag, TTag::EclBaseVanguard> {
     static constexpr bool value = true;
@@ -219,8 +227,12 @@ public:
                              "When restarting: should we try to initialize wells and groups from historical SCHEDULE section.");
         EWOMS_REGISTER_PARAM(TypeTag, int, EdgeWeightsMethod,
                              "Choose edge-weighing strategy: 0=uniform, 1=trans, 2=log(trans).");
+
+#if HAVE_OPENCL
         EWOMS_REGISTER_PARAM(TypeTag, int, NumJacobiBlocks,
                              "Number of blocks to be created for the Block-Jacobi preconditioner.");
+#endif
+
         EWOMS_REGISTER_PARAM(TypeTag, bool, OwnerCellsFirst,
                              "Order cells owned by rank before ghost/overlap cells.");
         EWOMS_REGISTER_PARAM(TypeTag, bool, SerialPartitioning,
@@ -245,7 +257,11 @@ public:
     {
         fileName_ = EWOMS_GET_PARAM(TypeTag, std::string, EclDeckFileName);
         edgeWeightsMethod_   = Dune::EdgeWeightMethod(EWOMS_GET_PARAM(TypeTag, int, EdgeWeightsMethod));
+
+#if HAVE_OPENCL
         numJacobiBlocks_ = EWOMS_GET_PARAM(TypeTag, int, NumJacobiBlocks);
+#endif
+
         ownersFirst_ = EWOMS_GET_PARAM(TypeTag, bool, OwnerCellsFirst);
         serialPartitioning_ = EWOMS_GET_PARAM(TypeTag, bool, SerialPartitioning);
         zoltanImbalanceTol_ = EWOMS_GET_PARAM(TypeTag, double, ZoltanImbalanceTol);
diff --git a/ebos/eclgenericcpgridvanguard.cc b/ebos/eclgenericcpgridvanguard.cc
index 95f51b4c2..e70c2af2b 100644
--- a/ebos/eclgenericcpgridvanguard.cc
+++ b/ebos/eclgenericcpgridvanguard.cc
@@ -144,12 +144,14 @@ doLoadBalance_(const Dune::EdgeWeightMethod            edgeWeightsMethod,
         // first cell of a well (e.g. for pressure).  Hence this is now
         // skipped.  Rank 0 had everything even before.
 
+#if HAVE_OPENCL
         if (partitionJacobiBlocks) {
             this->cell_part_ = this->grid_->
                 zoltanPartitionWithoutScatter(&wells, faceTrans.data(),
                                               numJacobiBlocks,
                                               zoltanImbalanceTol);
         }
+#endif // HAVE_OPENCL
     }
 }
 
diff --git a/ebos/eclgenericvanguard.hh b/ebos/eclgenericvanguard.hh
index 200ecb508..fb727a4c1 100644
--- a/ebos/eclgenericvanguard.hh
+++ b/ebos/eclgenericvanguard.hh
@@ -253,7 +253,13 @@ public:
      * \brief Number of blocks in the Block-Jacobi preconditioner.
      */
     int numJacobiBlocks() const
-    { return numJacobiBlocks_; }
+    {
+#if HAVE_OPENCL
+        return numJacobiBlocks_;
+#else
+        return 0;
+#endif
+    }
 
     /*!
      * \brief Parameter that decide if cells owned by rank are ordered before ghost cells.
@@ -329,7 +335,11 @@ protected:
     std::string caseName_;
     std::string fileName_;
     Dune::EdgeWeightMethod edgeWeightsMethod_;
-    int numJacobiBlocks_;
+
+#if HAVE_OPENCL
+    int numJacobiBlocks_{0};
+#endif  // HAVE_OPENCL
+
     bool ownersFirst_;
     bool serialPartitioning_;
     double zoltanImbalanceTol_;
diff --git a/opm/simulators/linalg/ISTLSolverEbos.hpp b/opm/simulators/linalg/ISTLSolverEbos.hpp
index 890fcc4ae..33edb89ad 100644
--- a/opm/simulators/linalg/ISTLSolverEbos.hpp
+++ b/opm/simulators/linalg/ISTLSolverEbos.hpp
@@ -213,7 +213,12 @@ namespace Opm
                 matrix_ = const_cast<Matrix*>(&M.istlMatrix());
 
                 // setup sparsity pattern for jacobi matrix for preconditioner (only used for openclSolver)
-                numJacobiBlocks_ = EWOMS_GET_PARAM(TypeTag, int, NumJacobiBlocks);
+#if HAVE_OPENCL
+                this->numJacobiBlocks_ = EWOMS_GET_PARAM(TypeTag, int, NumJacobiBlocks);
+#else
+                this->numJacobiBlocks_ = 0;
+#endif
+
                 useWellConn_ = EWOMS_GET_PARAM(TypeTag, bool, MatrixAddWellContributions);
                 if (numJacobiBlocks_ > 1) {
                     const auto wellsForConn = simulator_.vanguard().schedule().getWellsatEnd();