damariswriter: properly encapsulate Damaris

CMakeLists_files.cmake

@@ -623,7 +623,8 @@ if (Damaris_FOUND AND MPI_FOUND)
   list (APPEND PUBLIC_HEADER_FILES opm/simulators/utils/DamarisVar.hpp)
   list (APPEND PUBLIC_HEADER_FILES opm/simulators/utils/GridDataOutput.hpp
                                    opm/simulators/utils/GridDataOutput_impl.hpp)
-  list(APPEND MAIN_SOURCE_FILES opm/simulators/utils/DamarisVar.cpp
+  list(APPEND MAIN_SOURCE_FILES ebos/damariswriter.cc
+                                opm/simulators/utils/DamarisVar.cpp
                                 opm/simulators/utils/GridDataOutput.cpp)
 endif()
 

ebos/damariswriter.cc

@@ -0,0 +1,84 @@
+// -*- mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+// vi: set et ts=4 sw=4 sts=4:
+/*
+  Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
+  Copyright 2023 Inria, Bretagne–Atlantique Research Center
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 2 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+
+  Consult the COPYING file in the top-level source directory of this
+  module for the precise wording of the license and the list of
+  copyright holders.
+*/
+
+#include <config.h>
+#include <ebos/damariswriter.hh>
+
+#include <opm/common/OpmLog/OpmLog.hpp>
+
+#include <Damaris.h>
+#include <fmt/format.h>
+
+namespace Opm::DamarisOutput {
+
+int setPosition(const char* field, int rank, int64_t pos)
+{
+    int dam_err = damaris_set_position(field, &pos);
+    if (dam_err != DAMARIS_OK) {
+        OpmLog::error(fmt::format("damariswriter::writeOutput()       : ( rank:{})"
+                                  "damaris_set_position({}, ...), Damaris Error: {}  ",
+                                  rank, field, damaris_error_string(dam_err)));
+    }
+
+    return dam_err;
+}
+
+int setParameter(const char* field, int rank, int value)
+{
+    int dam_err = damaris_parameter_set(field, &value, sizeof(int));
+    if (dam_err != DAMARIS_OK) {
+        OpmLog::error(fmt::format("(rank:{}) Damaris library produced an error result for "
+                                  "damaris_parameter_set(\"{}\",...)", rank, field));
+    }
+
+    return dam_err;
+}
+
+int write(const char* field, int rank, const void* data)
+{
+    int dam_err = damaris_write(field, data);
+    if (dam_err != DAMARIS_OK) {
+        OpmLog::error(fmt::format("damariswriter::writeOutput()       : ( rank:{}) "
+                                  "damaris_write({}, ...), Damaris Error: {}  ",
+                                  rank, field, damaris_error_string(dam_err)));
+    }
+
+    return dam_err;
+}
+
+int endIteration(int rank)
+{
+    int dam_err =  damaris_end_iteration();
+    if (dam_err != DAMARIS_OK) {
+        OpmLog::error(fmt::format("damariswriter::writeOutput()       : ( rank:{}) "
+                                  "damaris_end_iteration(), Damaris Error: {}  ",
+                                  rank, damaris_error_string(dam_err)));
+    }
+
+    return dam_err;
+}
+
+}

ebos/damariswriter.hh

@@ -47,15 +47,24 @@
 
 #include <fmt/format.h>
 
-#include <Damaris.h>
+#include <algorithm>
-
 #include <limits>
 #include <memory>
+#include <numeric>
 #include <stdexcept>
 #include <string>
 
 namespace Opm {
 
+namespace DamarisOutput {
+
+int endIteration(int rank);
+int setParameter(const char* field, int rank, int value);
+int setPosition(const char* field, int rank, int64_t pos);
+int write(const char* field, int rank, const void* data);
+
+}
+
 /*!
  * \ingroup EclBlackOilSimulator
  *
@@ -211,28 +220,12 @@ public:
 
             if (this->damarisOutputModule_->getPRESSURE_ptr() != nullptr) 
             {
-                int64_t temp_int64_t[1];
+                dam_err_ = DamarisOutput::setPosition("PRESSURE", rank_,
-                temp_int64_t[0] = static_cast<int64_t>(this->elements_rank_offsets_[rank_]);
+                                                      this->elements_rank_offsets_[rank_]);
-                dam_err_ = damaris_set_position("PRESSURE", temp_int64_t);
+                dam_err_ = DamarisOutput::write("PRESSURE", rank_,
-                if (dam_err_ != DAMARIS_OK) {
+                                                this->damarisOutputModule_->getPRESSURE_ptr());
-                    OpmLog::error(fmt::format("damariswriter::writeOutput()       : ( rank:{})"
-                                              "damaris_set_position(PRESSURE, ...), Damaris Error: {}  ",
-                                              rank_, damaris_error_string(dam_err_) ));
-                }
 
-                dam_err_ = damaris_write("PRESSURE", (void*)this->damarisOutputModule_->getPRESSURE_ptr());
+                dam_err_ =  DamarisOutput::endIteration(rank_);
-                if (dam_err_ != DAMARIS_OK) {
-                   OpmLog::error(fmt::format("damariswriter::writeOutput()       : ( rank:{}) "
-                                             "damaris_write(PRESSURE, ...), Damaris Error: {}  ",
-                                             rank_, damaris_error_string(dam_err_) ));
-                }
-
-                dam_err_ =  damaris_end_iteration();
-                if (dam_err_ != DAMARIS_OK) {
-                    OpmLog::error(fmt::format("damariswriter::writeOutput()       : ( rank:{}) "
-                                              "damaris_end_iteration(), Damaris Error: {}  ",
-                                              rank_, damaris_error_string(dam_err_) ));
-                }
             }
          } // end of ! isSubstep
     }
@@ -257,25 +250,16 @@ private:
     {
         // GLOBAL_CELL_INDEX is used to reorder variable data when writing to disk 
         // This is enabled using select-file="GLOBAL_CELL_INDEX" in the <variable> XML tag
-        if ( this->collectToIORank_.isParallel() ){
+        if (this->collectToIORank_.isParallel()) {
-            const std::vector<int>& local_to_global =  this->collectToIORank_.localIdxToGlobalIdxMapping(); 
+            const std::vector<int>& local_to_global =  this->collectToIORank_.localIdxToGlobalIdxMapping();
-            dam_err_ = damaris_write("GLOBAL_CELL_INDEX", local_to_global.data());
+            dam_err_ = DamarisOutput::write("GLOBAL_CELL_INDEX", rank_, local_to_global.data());
         } else {
             std::vector<int> local_to_global_filled ;
             local_to_global_filled.resize(this->numElements_) ;
-            for (int i = 0 ; i < this->numElements_ ; i++)
+            std::iota(local_to_global_filled.begin(), local_to_global_filled.end(), 0);
-            {
+            dam_err_ = DamarisOutput::write("GLOBAL_CELL_INDEX", rank_, local_to_global_filled.data());
-                local_to_global_filled[i] = i ;
-            }
-            dam_err_ = damaris_write("GLOBAL_CELL_INDEX", local_to_global_filled.data());
         }
 
-        if (dam_err_ != DAMARIS_OK) {
-            OpmLog::error(fmt::format("damariswriter::writeOutput()       :"
-                                      "( rank:{}) damaris_write(GLOBAL_CELL_INDEX, ...), Damaris Error: {}  ",  
-                                      rank_, damaris_error_string(dam_err_) ));
-        }
-        
         // This is an example of writing to the Damaris shared memory directly (i.e. not using 
         // damaris_write() to copy data there)
         // We will add the MPI rank value directly into shared memory using the DamarisVar 
@@ -284,14 +268,12 @@ private:
         DamarisVarInt mpi_rank_var_test(1, {std::string("n_elements_local")},  std::string("MPI_RANK"), rank_);
         mpi_rank_var_test.setDamarisParameterAndShmem( {this->numElements_ } ) ;
         // Fill the created memory area
-        for (int i = 0 ; i < this->numElements_; i++ )
+        std::fill(mpi_rank_var_test.data(), mpi_rank_var_test.data() + numElements_, rank_);
-        {
-            mpi_rank_var_test.data()[i] = rank_ ;  // write the rank vaue to the shared memory area.
-        }
-       
     }
 
-    void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid, std::vector<unsigned long long>& elements_rank_offsets)
+    void setupDamarisWritingPars(Parallel::Communication comm,
+                                 const int n_elements_local_grid,
+                                 std::vector<unsigned long long>& elements_rank_offsets)
     {
         // one for each rank -- to be gathered from each client rank
         std::vector<unsigned long long> elements_rank_sizes(nranks_); 
@@ -319,21 +301,11 @@ private:
         // Set the paramater so that the Damaris servers can allocate the correct amount of memory for the variabe
         // Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
         // ToDo: Do we need to check that local ranks are 0 based ?
-        int temp_int = static_cast<int>(elements_rank_sizes[rank_]);
+        dam_err_ = DamarisOutput::setParameter("n_elements_local", rank_, elements_rank_sizes[rank_]);
-        dam_err_ = damaris_parameter_set("n_elements_local", &temp_int, sizeof(int));
-        if (dam_err_ != DAMARIS_OK) {
-            OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
-                          "damaris_parameter_set(\"n_elements_local\", &temp_int, sizeof(int));");
-        }
         // Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
         // ToDo: Do we need to check that n_elements_global_max will fit in a C int type (INT_MAX)
         if( n_elements_global_max <= std::numeric_limits<int>::max() ) {
-            temp_int = static_cast<int>(n_elements_global_max);
+            dam_err_ = DamarisOutput::setParameter("n_elements_total", rank_, n_elements_global_max);
-            dam_err_ = damaris_parameter_set("n_elements_total", &temp_int, sizeof(int));
-            if (dam_err_ != DAMARIS_OK) {
-                OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
-                              "damaris_parameter_set(\"n_elements_total\", &temp_int, sizeof(int));");
-            }
         } else {
             OpmLog::error(fmt::format("( rank:{} ) The size of the global array ({}) is"
                                       "greater than what a Damaris paramater type supports ({}).  ", 
@@ -344,28 +316,15 @@ private:
 
         // Use damaris_set_position to set the offset in the global size of the array.
         // This is used so that output functionality (e.g. HDF5Store) knows global offsets of the data of the ranks
-        int64_t temp_int64_t[1];
+        dam_err_ = DamarisOutput::setPosition("PRESSURE", rank_, elements_rank_offsets[rank_]);
-        temp_int64_t[0] = static_cast<int64_t>(elements_rank_offsets[rank_]);
+        dam_err_ = DamarisOutput::setPosition("GLOBAL_CELL_INDEX", rank_, elements_rank_offsets[rank_]);
-        dam_err_ = damaris_set_position("PRESSURE", temp_int64_t);
-        if (dam_err_ != DAMARIS_OK) {
-            OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
-                          "damaris_set_position(\"PRESSURE\", temp_int64_t);");
-        }
-        dam_err_ = damaris_set_position("GLOBAL_CELL_INDEX", temp_int64_t);
-        if (dam_err_ != DAMARIS_OK) {
-            OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
-                          "damaris_set_position(\"GLOBAL_CELL_INDEX\", temp_int64_t);");
-        }
 
         // Set the size of the MPI variable
         DamarisVarInt mpi_rank_var(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_)  ;
-        mpi_rank_var.setDamarisPosition({*temp_int64_t}) ;
+        mpi_rank_var.setDamarisPosition({static_cast<int64_t>(elements_rank_offsets[rank_])});
-
     }
-    
 
-
+    void writeDamarisGridOutput()
-    void writeDamarisGridOutput( void )
     {
         const auto& gridView = simulator_.gridView();
         Opm::GridDataOutput::SimMeshDataAccessor geomData(gridView, Dune::Partitions::interior) ;

opm/simulators/flow/Main.cpp

@@ -31,6 +31,7 @@
 #include <opm/simulators/utils/readDeck.hpp>
 
 #if HAVE_DAMARIS
+#include <Damaris.h>
 #include <opm/simulators/utils/DamarisOutputModule.hpp>
 #endif