diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake
index 2654a5863..41552add2 100644
--- a/CMakeLists_files.cmake
+++ b/CMakeLists_files.cmake
@@ -623,7 +623,8 @@ if (Damaris_FOUND AND MPI_FOUND)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/utils/DamarisVar.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/utils/GridDataOutput.hpp
opm/simulators/utils/GridDataOutput_impl.hpp)
- list(APPEND MAIN_SOURCE_FILES opm/simulators/utils/DamarisVar.cpp
+ list(APPEND MAIN_SOURCE_FILES ebos/damariswriter.cc
+ opm/simulators/utils/DamarisVar.cpp
opm/simulators/utils/GridDataOutput.cpp)
endif()
diff --git a/ebos/damariswriter.cc b/ebos/damariswriter.cc
new file mode 100644
index 000000000..8f6291d94
--- /dev/null
+++ b/ebos/damariswriter.cc
@@ -0,0 +1,84 @@
+// -*- mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+// vi: set et ts=4 sw=4 sts=4:
+/*
+ Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
+ Copyright 2023 Inria, Bretagne–Atlantique Research Center
+
+ This file is part of the Open Porous Media project (OPM).
+
+ OPM is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ (at your option) any later version.
+
+ OPM is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OPM. If not, see .
+
+ Consult the COPYING file in the top-level source directory of this
+ module for the precise wording of the license and the list of
+ copyright holders.
+*/
+
+#include
+#include
+
+#include
+
+#include
+#include
+
+namespace Opm::DamarisOutput {
+
+int setPosition(const char* field, int rank, int64_t pos)
+{
+ int dam_err = damaris_set_position(field, &pos);
+ if (dam_err != DAMARIS_OK) {
+ OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})"
+ "damaris_set_position({}, ...), Damaris Error: {} ",
+ rank, field, damaris_error_string(dam_err)));
+ }
+
+ return dam_err;
+}
+
+int setParameter(const char* field, int rank, int value)
+{
+ int dam_err = damaris_parameter_set(field, &value, sizeof(int));
+ if (dam_err != DAMARIS_OK) {
+ OpmLog::error(fmt::format("(rank:{}) Damaris library produced an error result for "
+ "damaris_parameter_set(\"{}\",...)", rank, field));
+ }
+
+ return dam_err;
+}
+
+int write(const char* field, int rank, const void* data)
+{
+ int dam_err = damaris_write(field, data);
+ if (dam_err != DAMARIS_OK) {
+ OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
+ "damaris_write({}, ...), Damaris Error: {} ",
+ rank, field, damaris_error_string(dam_err)));
+ }
+
+ return dam_err;
+}
+
+int endIteration(int rank)
+{
+ int dam_err = damaris_end_iteration();
+ if (dam_err != DAMARIS_OK) {
+ OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
+ "damaris_end_iteration(), Damaris Error: {} ",
+ rank, damaris_error_string(dam_err)));
+ }
+
+ return dam_err;
+}
+
+}
diff --git a/ebos/damariswriter.hh b/ebos/damariswriter.hh
index e4377d86d..b5326d326 100644
--- a/ebos/damariswriter.hh
+++ b/ebos/damariswriter.hh
@@ -47,15 +47,24 @@
#include
-#include
-
+#include
#include
#include
+#include
#include
#include
namespace Opm {
+namespace DamarisOutput {
+
+int endIteration(int rank);
+int setParameter(const char* field, int rank, int value);
+int setPosition(const char* field, int rank, int64_t pos);
+int write(const char* field, int rank, const void* data);
+
+}
+
/*!
* \ingroup EclBlackOilSimulator
*
@@ -211,28 +220,12 @@ public:
if (this->damarisOutputModule_->getPRESSURE_ptr() != nullptr)
{
- int64_t temp_int64_t[1];
- temp_int64_t[0] = static_cast(this->elements_rank_offsets_[rank_]);
- dam_err_ = damaris_set_position("PRESSURE", temp_int64_t);
- if (dam_err_ != DAMARIS_OK) {
- OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})"
- "damaris_set_position(PRESSURE, ...), Damaris Error: {} ",
- rank_, damaris_error_string(dam_err_) ));
- }
+ dam_err_ = DamarisOutput::setPosition("PRESSURE", rank_,
+ this->elements_rank_offsets_[rank_]);
+ dam_err_ = DamarisOutput::write("PRESSURE", rank_,
+ this->damarisOutputModule_->getPRESSURE_ptr());
- dam_err_ = damaris_write("PRESSURE", (void*)this->damarisOutputModule_->getPRESSURE_ptr());
- if (dam_err_ != DAMARIS_OK) {
- OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
- "damaris_write(PRESSURE, ...), Damaris Error: {} ",
- rank_, damaris_error_string(dam_err_) ));
- }
-
- dam_err_ = damaris_end_iteration();
- if (dam_err_ != DAMARIS_OK) {
- OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
- "damaris_end_iteration(), Damaris Error: {} ",
- rank_, damaris_error_string(dam_err_) ));
- }
+ dam_err_ = DamarisOutput::endIteration(rank_);
}
} // end of ! isSubstep
}
@@ -257,25 +250,16 @@ private:
{
// GLOBAL_CELL_INDEX is used to reorder variable data when writing to disk
// This is enabled using select-file="GLOBAL_CELL_INDEX" in the XML tag
- if ( this->collectToIORank_.isParallel() ){
- const std::vector& local_to_global = this->collectToIORank_.localIdxToGlobalIdxMapping();
- dam_err_ = damaris_write("GLOBAL_CELL_INDEX", local_to_global.data());
+ if (this->collectToIORank_.isParallel()) {
+ const std::vector& local_to_global = this->collectToIORank_.localIdxToGlobalIdxMapping();
+ dam_err_ = DamarisOutput::write("GLOBAL_CELL_INDEX", rank_, local_to_global.data());
} else {
std::vector local_to_global_filled ;
local_to_global_filled.resize(this->numElements_) ;
- for (int i = 0 ; i < this->numElements_ ; i++)
- {
- local_to_global_filled[i] = i ;
- }
- dam_err_ = damaris_write("GLOBAL_CELL_INDEX", local_to_global_filled.data());
+ std::iota(local_to_global_filled.begin(), local_to_global_filled.end(), 0);
+ dam_err_ = DamarisOutput::write("GLOBAL_CELL_INDEX", rank_, local_to_global_filled.data());
}
- if (dam_err_ != DAMARIS_OK) {
- OpmLog::error(fmt::format("damariswriter::writeOutput() :"
- "( rank:{}) damaris_write(GLOBAL_CELL_INDEX, ...), Damaris Error: {} ",
- rank_, damaris_error_string(dam_err_) ));
- }
-
// This is an example of writing to the Damaris shared memory directly (i.e. not using
// damaris_write() to copy data there)
// We will add the MPI rank value directly into shared memory using the DamarisVar
@@ -284,14 +268,12 @@ private:
DamarisVarInt mpi_rank_var_test(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_);
mpi_rank_var_test.setDamarisParameterAndShmem( {this->numElements_ } ) ;
// Fill the created memory area
- for (int i = 0 ; i < this->numElements_; i++ )
- {
- mpi_rank_var_test.data()[i] = rank_ ; // write the rank vaue to the shared memory area.
- }
-
+ std::fill(mpi_rank_var_test.data(), mpi_rank_var_test.data() + numElements_, rank_);
}
- void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid, std::vector& elements_rank_offsets)
+ void setupDamarisWritingPars(Parallel::Communication comm,
+ const int n_elements_local_grid,
+ std::vector& elements_rank_offsets)
{
// one for each rank -- to be gathered from each client rank
std::vector elements_rank_sizes(nranks_);
@@ -319,21 +301,11 @@ private:
// Set the paramater so that the Damaris servers can allocate the correct amount of memory for the variabe
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that local ranks are 0 based ?
- int temp_int = static_cast(elements_rank_sizes[rank_]);
- dam_err_ = damaris_parameter_set("n_elements_local", &temp_int, sizeof(int));
- if (dam_err_ != DAMARIS_OK) {
- OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
- "damaris_parameter_set(\"n_elements_local\", &temp_int, sizeof(int));");
- }
+ dam_err_ = DamarisOutput::setParameter("n_elements_local", rank_, elements_rank_sizes[rank_]);
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that n_elements_global_max will fit in a C int type (INT_MAX)
if( n_elements_global_max <= std::numeric_limits::max() ) {
- temp_int = static_cast(n_elements_global_max);
- dam_err_ = damaris_parameter_set("n_elements_total", &temp_int, sizeof(int));
- if (dam_err_ != DAMARIS_OK) {
- OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
- "damaris_parameter_set(\"n_elements_total\", &temp_int, sizeof(int));");
- }
+ dam_err_ = DamarisOutput::setParameter("n_elements_total", rank_, n_elements_global_max);
} else {
OpmLog::error(fmt::format("( rank:{} ) The size of the global array ({}) is"
"greater than what a Damaris paramater type supports ({}). ",
@@ -344,28 +316,15 @@ private:
// Use damaris_set_position to set the offset in the global size of the array.
// This is used so that output functionality (e.g. HDF5Store) knows global offsets of the data of the ranks
- int64_t temp_int64_t[1];
- temp_int64_t[0] = static_cast(elements_rank_offsets[rank_]);
- dam_err_ = damaris_set_position("PRESSURE", temp_int64_t);
- if (dam_err_ != DAMARIS_OK) {
- OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
- "damaris_set_position(\"PRESSURE\", temp_int64_t);");
- }
- dam_err_ = damaris_set_position("GLOBAL_CELL_INDEX", temp_int64_t);
- if (dam_err_ != DAMARIS_OK) {
- OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
- "damaris_set_position(\"GLOBAL_CELL_INDEX\", temp_int64_t);");
- }
+ dam_err_ = DamarisOutput::setPosition("PRESSURE", rank_, elements_rank_offsets[rank_]);
+ dam_err_ = DamarisOutput::setPosition("GLOBAL_CELL_INDEX", rank_, elements_rank_offsets[rank_]);
// Set the size of the MPI variable
DamarisVarInt mpi_rank_var(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_) ;
- mpi_rank_var.setDamarisPosition({*temp_int64_t}) ;
-
+ mpi_rank_var.setDamarisPosition({static_cast(elements_rank_offsets[rank_])});
}
-
-
- void writeDamarisGridOutput( void )
+ void writeDamarisGridOutput()
{
const auto& gridView = simulator_.gridView();
Opm::GridDataOutput::SimMeshDataAccessor geomData(gridView, Dune::Partitions::interior) ;
diff --git a/opm/simulators/flow/Main.cpp b/opm/simulators/flow/Main.cpp
index a9de12950..4d16bdad0 100644
--- a/opm/simulators/flow/Main.cpp
+++ b/opm/simulators/flow/Main.cpp
@@ -31,6 +31,7 @@
#include
#if HAVE_DAMARIS
+#include
#include
#endif