From e1dd2bf148bcfad4d9d1a4adad826413e9a29e45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Atgeirr=20Fl=C3=B8=20Rasmussen?= <atgeirr@sintef.no>
Date: Tue, 22 Aug 2023 16:33:18 +0200
Subject: [PATCH] Add forceSerial parameter to ISTLSolverEbos.

Used for single-domain solves.
---
 opm/simulators/flow/BlackoilModelEbosNldd.hpp |  3 ++-
 opm/simulators/linalg/ISTLSolverEbos.cpp      |  3 ++-
 opm/simulators/linalg/ISTLSolverEbos.hpp      | 10 +++++++---
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/opm/simulators/flow/BlackoilModelEbosNldd.hpp b/opm/simulators/flow/BlackoilModelEbosNldd.hpp
index 5fb2a348b..1dbe100fe 100644
--- a/opm/simulators/flow/BlackoilModelEbosNldd.hpp
+++ b/opm/simulators/flow/BlackoilModelEbosNldd.hpp
@@ -168,7 +168,8 @@ public:
                 loc_param.linear_solver_reduction_ = 1e-2;
             }
             loc_param.linear_solver_print_json_definition_ = false;
-            domain_linsolvers_.emplace_back(model_.ebosSimulator(), loc_param);
+            const bool force_serial = true;
+            domain_linsolvers_.emplace_back(model_.ebosSimulator(), loc_param, force_serial);
         }
 
         assert(int(domains_.size()) == num_domains);
diff --git a/opm/simulators/linalg/ISTLSolverEbos.cpp b/opm/simulators/linalg/ISTLSolverEbos.cpp
index 25b3d2cdd..241df3fb0 100644
--- a/opm/simulators/linalg/ISTLSolverEbos.cpp
+++ b/opm/simulators/linalg/ISTLSolverEbos.cpp
@@ -124,11 +124,12 @@ void FlexibleSolverInfo<Matrix,Vector,Comm>::create(const Matrix& matrix,
                                                     const PropertyTree& prm,
                                                     std::size_t pressureIndex,
                                                     std::function<Vector()> trueFunc,
+                                                    const bool forceSerial,
                                                     [[maybe_unused]] Comm& comm)
 
 {
     // Write sizes of linear systems on all ranks to debug log.
-    {
+    if (!forceSerial) {
 #if HAVE_MPI
         auto basic_comm = comm.communicator();
 #else
diff --git a/opm/simulators/linalg/ISTLSolverEbos.hpp b/opm/simulators/linalg/ISTLSolverEbos.hpp
index 6f26d8c07..6118d536d 100644
--- a/opm/simulators/linalg/ISTLSolverEbos.hpp
+++ b/opm/simulators/linalg/ISTLSolverEbos.hpp
@@ -103,6 +103,7 @@ struct FlexibleSolverInfo
                 const PropertyTree& prm,
                 std::size_t pressureIndex,
                 std::function<Vector()> trueFunc,
+                const bool forceSerial,
                 Comm& comm);
 
     std::unique_ptr<AbstractSolverType> solver_;
@@ -177,13 +178,14 @@ std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
         /// \param[in] simulator   The opm-models simulator object
         /// \param[in] parameters  Explicit parameters for solver setup, do not
         ///                        read them from command line parameters.
-        ISTLSolverEbos(const Simulator& simulator, const FlowLinearSolverParameters& parameters)
+        ISTLSolverEbos(const Simulator& simulator, const FlowLinearSolverParameters& parameters, bool forceSerial = false)
             : simulator_(simulator),
               iterations_( 0 ),
               calls_( 0 ),
               converged_(false),
               matrix_(nullptr),
-              parameters_(parameters)
+              parameters_(parameters),
+              forceSerial_(forceSerial)
         {
             initialize();
         }
@@ -377,7 +379,7 @@ std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
 
         bool isParallel() const {
 #if HAVE_MPI
-            return comm_->communicator().size() > 1;
+            return !forceSerial_ && comm_->communicator().size() > 1;
 #else
             return false;
 #endif
@@ -403,6 +405,7 @@ std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
                                        prm_,
                                        pressureIndex,
                                        trueFunc,
+                                       forceSerial_,
                                        *comm_);
             }
             else
@@ -502,6 +505,7 @@ std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
         bool useWellConn_;
 
         FlowLinearSolverParameters parameters_;
+        bool forceSerial_ = false;
         PropertyTree prm_;
 
         std::shared_ptr< CommunicationType > comm_;