Improve NLDD reporting with per-domain solve statistics

opm/simulators/flow/BlackoilModel.hpp

@@ -262,6 +262,9 @@ public:
     /// return the statistics of local solves accumulated for this rank
     SimulatorReport localAccumulatedReports() const;
 
+    /// return the statistics of local solves accumulated for each domain on this rank
+    std::vector<SimulatorReport> domainAccumulatedReports() const;
+
     const std::vector<StepReport>& stepReports() const
     { return convergence_reports_; }
 

opm/simulators/flow/BlackoilModelNldd.hpp

@@ -204,6 +204,8 @@ public:
 
         assert(int(domains_.size()) == num_domains);
 
+        domain_reports_accumulated_.resize(num_domains);
+
         // Print domain distribution summary
         printDomainDistributionSummary(partition_vector);
     }
@@ -299,7 +301,8 @@ public:
         int& num_domains = counts[3];
         {
             auto step_newtons = 0;
-            for (const auto& dr : domain_reports) {
+            for (size_t i = 0; i < domain_reports.size(); ++i) {
+                const auto& dr = domain_reports[i];
                 if (dr.converged) {
                     ++num_converged;
                     if (dr.total_newton_iterations == 0) {
@@ -307,6 +310,9 @@ public:
                     }
                 }
                 step_newtons += dr.total_newton_iterations;
+                // Accumulate local reports per domain
+                domain_reports_accumulated_[i] += dr;
+                // Accumulate local reports per rank
                 local_reports_accumulated_ += dr;
             }
             num_local_newtons = step_newtons;
@@ -380,6 +386,12 @@ public:
         return local_reports_accumulated_;
     }
 
+    /// return the statistics of local solves accumulated for each domain on this rank
+    const std::vector<SimulatorReport>& domainAccumulatedReports() const
+    {
+        return domain_reports_accumulated_;
+    }
+
     /// Write the partition vector to a file in ResInsight compatible format for inspection
     /// and a partition file for each rank, that can be used as input for OPM.
     void writePartitions(const std::filesystem::path& odir) const
@@ -1165,6 +1177,7 @@ private:
     std::vector<std::unique_ptr<Mat>> domain_matrices_; //!< Vector of matrix operator for each subdomain
     std::vector<ISTLSolverType> domain_linsolvers_; //!< Vector of linear solvers for each domain
     SimulatorReport local_reports_accumulated_; //!< Accumulated convergence report for subdomain solvers per rank
+    std::vector<SimulatorReport> domain_reports_accumulated_; //!< Accumulated convergence reports per domain
     int rank_ = 0; //!< MPI rank of this process
 };
 

opm/simulators/flow/BlackoilModel_impl.hpp

@@ -985,6 +985,15 @@ localAccumulatedReports() const
                            : SimulatorReport{};
 }
 
+template <class TypeTag>
+std::vector<SimulatorReport>
+BlackoilModel<TypeTag>::
+domainAccumulatedReports() const
+{
+    return hasNlddSolver() ? nlddSolver_->domainAccumulatedReports()
+                           : std::vector<SimulatorReport>{};
+}
+
 template <class TypeTag>
 void
 BlackoilModel<TypeTag>::

opm/simulators/flow/FlowMain.hpp

@@ -392,18 +392,42 @@ namespace Opm {
         void runSimulatorAfterSim_(SimulatorReport &report)
         {
             if (simulator_->model().hasNlddSolver()) {
-                // Create a deferred logger and add the report with rank as tag
-                DeferredLogger local_log;
-                std::ostringstream ss;
-                ss << "======  Accumulated local solve data for rank " << mpi_rank_ << " ======\n";
-                simulator_->model().localAccumulatedReports().reportNLDD(ss);
-                // Use rank number as tag to ensure correct ordering
-                local_log.debug(fmt::format("{:05d}", mpi_rank_), ss.str());
+                {
+                    // Create a deferred logger and add the report with rank as tag
+                    DeferredLogger local_log;
+                    std::ostringstream ss;
 
-                // Gather all logs and output them in sorted order
-                auto global_log = gatherDeferredLogger(local_log, FlowGenericVanguard::comm());
-                if (this->output_cout_) {
-                    global_log.logMessages();
+                    // Accumulate reports per domain
+                    const auto& domain_reports = simulator_->model().domainAccumulatedReports();
+                    for (size_t i = 0; i < domain_reports.size(); ++i) {
+                        const auto& dr = domain_reports[i];
+                        ss << "======  Accumulated local solve data for domain " << i << " on rank " << mpi_rank_ << " ======\n";
+                        dr.reportNLDD(ss);
+                        // Use combined rank and domain index as tag to ensure global uniqueness and correct ordering
+                        local_log.debug(fmt::format("R{:05d}D{:05d}", mpi_rank_, i), ss.str());
+                        ss.str(""); // Clear the stringstream
+                        ss.clear(); // Clear any error flags
+                    }
+                    // Gather all logs and output them in sorted order
+                    auto global_log = gatherDeferredLogger(local_log, FlowGenericVanguard::comm());
+                    if (this->output_cout_) {
+                        global_log.logMessages();
+                    }
+                }
+                {
+                    // Create a deferred logger and add the report with rank as tag
+                    DeferredLogger local_log;
+                    std::ostringstream ss;
+                    ss << "======  Accumulated local solve data for rank " << mpi_rank_ << " ======\n";
+                    simulator_->model().localAccumulatedReports().reportNLDD(ss);
+                    // Use rank number as tag to ensure correct ordering
+                    local_log.debug(fmt::format("{:05d}", mpi_rank_), ss.str());
+
+                    // Gather all logs and output them in sorted order
+                    auto global_log = gatherDeferredLogger(local_log, FlowGenericVanguard::comm());
+                    if (this->output_cout_) {
+                        global_log.logMessages();
+                    }
                 }
             }