OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4889 from jcbowden/damariswriter-for-sim-fields-v5

Browse Source 
Geometric mesh data added for Damaris in-situ visualisation
This commit is contained in:
Atgeirr Flø Rasmussen 2023-12-15 10:18:00 +01:00 committed by GitHub
commit 1dbd971710
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 2189 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists_files.cmake
View File

@ -609,7 +609,11 @@ list (APPEND PUBLIC_HEADER_FILES
)
if (Damaris_FOUND AND MPI_FOUND)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/utils/DamarisOutputModule.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/utils/DamarisKeywords.hpp)
list (APPEND PUBLIC_HEADER_FILES ebos/damariswriter.hh)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/utils/DamarisVar.hpp)
list (APPEND PUBLIC_HEADER_FILES opm/simulators/utils/GridDataOutput.hpp)
endif()
if(HDF5_FOUND)

265
ebos/damariswriter.hh
View File

@ -50,10 +50,15 @@
#include <limits>
#include <stdexcept>
#include <string>
#include <memory>
#include <omp.h>
#include <fmt/format.h>
#include <Damaris.h>
#include <opm/simulators/utils/GridDataOutput.hpp>
#include <opm/simulators/utils/DamarisVar.hpp>
namespace Opm::Properties {
@ -62,7 +67,51 @@ struct EnableDamarisOutput {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct EnableDamarisOutputCollective {
struct DamarisOutputHdfCollective {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisSaveMeshToHdf {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisSaveToHdf {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisPythonScript {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisPythonParaviewScript {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisSimName {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisDedicatedCores {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisDedicatedNodes {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisSharedMemoryName {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisSharedMemorySizeBytes {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisLogLevel {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisDaskFile {
using type = UndefinedProperty;
};
} // namespace Opm::Properties
@ -75,7 +124,7 @@ namespace Opm {
* \brief Collects necessary output values and pass them to Damaris server processes.
*
* Currently only passing through PRESSURE, GLOBAL_CELL_INDEX and MPI_RANK information.
* This clss will be enhanced to pass through the 3D mesh information to Damaris to enable
* This class now passes through the 3D mesh information to Damaris to enable
* in situ visualization via Paraview or Ascent. And developed so that variables specified
* through the Eclipse input deck will be available to Damaris.
*/
@ -98,12 +147,50 @@ class DamarisWriter : public EclGenericWriter<GetPropType<TypeTag, Properties::G
using ElementMapper = GetPropType<TypeTag, Properties::ElementMapper>;
using BaseType = EclGenericWriter<Grid,EquilGrid,GridView,ElementMapper,Scalar>;
using DamarisVarInt = Opm::DamarisOutput::DamarisVar<int> ;
using DamarisVarChar = Opm::DamarisOutput::DamarisVar<char> ;
using DamarisVarDbl = Opm::DamarisOutput::DamarisVar<double> ;
public:
static void registerParameters()
{
EWOMS_REGISTER_PARAM(TypeTag, bool, EnableDamarisOutputCollective,
"Write output via Damaris using parallel HDF5 to get single file per timestep instead of one per Damaris core.");
EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisOutputHdfCollective,
"Write output via Damaris using parallel HDF5 to get single file and dataset per timestep instead of one per Damaris \n \
core with multiple datasets.");
EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisSaveToHdf,
"Set to false to prevent output to HDF5. Uses collective output by default or set --enable-damaris-collective=false to\n \
use file per core (file per Damaris server).");
EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisSaveMeshToHdf,
"Saves the mesh data to the HDF5 file (1st iteration only). Will set --damaris-output-hdf-collective to false \n \
so will use file per core (file per Damaris server) output (global sizes and offset values \n \
of mesh variables are not being provided as yet).");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisPythonScript,
"Set to the path and filename of a Python script to run on Damaris server resources with access to OPM flow data.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisPythonParaviewScript,
"Set to the path and filename of a Paraview Python script to run on Paraview Catalyst (1 or 2) on Damaris server \n \
resources with access to OPM flow data.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSimName,
"The name of the simulation to be used by Damaris. If empty (the default) then Damaris uses \"opm-sim-<random-number>\". \n \
This name is used for the Damaris HDF5 file name prefix. Make unique if writing to the same output directory.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisLogLevel,
"The log level for the Damaris logging system (boost log based). \n \
Levels are: [trace, debug, info, warning, error, fatal]. Currently debug and info are useful. ");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisDaskFile,
"The name of a Dask json configuration file (if using Dask for processing).");
EWOMS_REGISTER_PARAM(TypeTag, int, DamarisDedicatedCores,
"Set the number of dedicated cores (MPI processes) that should be used for Damaris processing (per node). \n \
Must divide evenly into the number of simulation ranks (client ranks).");
EWOMS_REGISTER_PARAM(TypeTag, int, DamarisDedicatedNodes,
"Set the number of dedicated nodes (full nodes) that should be used for Damaris processing (per simulation). \n \
Must divide evenly into the number of simulation nodes.");
EWOMS_REGISTER_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes,
"Set the size of the shared memory buffer used for IPC between the simulation and the Damaris resources. \n \
Needs to hold all the variables published, possibly over multiple simulation iterations.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSharedMemoryName,
"The name of the shared memory area to be used by Damaris for the current. If empty (the default) then Damaris uses \"opm-damaris-<random-string>\". \n \
This name should be unique if multiple simulations are running on the same node/server as it is used for the Damaris shmem name and by the Python Dask \n \
library to locate sections of variables.");
}
// The Simulator object should preferably have been const - the
@ -175,8 +262,12 @@ public:
// sets data for non-time-varying variables MPI_RANK and GLOBAL_CELL_INDEX
this->setGlobalIndexForDamaris() ;
// Currently by default we assume static grid (unchanging through the simulation)
// Set damarisUpdate_ to true if we want to update the geometry to sent to Damaris
// Set the geometry data for the mesh model.
// this function writes the mesh data directly to Damaris shared memory using Opm::DamarisOutput::DamarisVar objects.
this->writeDamarisGridOutput() ;
// Currently by default we assume a static mesh grid (the geometry unchanging through the simulation)
// Set damarisUpdate_ to true if we want to update the geometry sent to Damaris
this->damarisUpdate_ = false;
}
@ -185,18 +276,24 @@ public:
int64_t temp_int64_t[1];
temp_int64_t[0] = static_cast<int64_t>(this->elements_rank_offsets_[rank_]);
dam_err_ = damaris_set_position("PRESSURE", temp_int64_t);
if (dam_err_ != DAMARIS_OK && rank_ == 0) {
OpmLog::error(fmt::format("ERORR: damariswriter::writeOutput() : ( rank:{}) damaris_set_position(PRESSURE, ...), Damaris Error: {} ", rank_, damaris_error_string(dam_err_) ));
if (dam_err_ != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})"
"damaris_set_position(PRESSURE, ...), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) ));
}
dam_err_ = damaris_write("PRESSURE", (void*)this->damarisOutputModule_->getPRESSURE_ptr());
if (dam_err_ != DAMARIS_OK) {
OpmLog::error(fmt::format("ERORR: damariswriter::writeOutput() : ( rank:{}) damaris_write(PRESSURE, ...), Damaris Error: {} ", rank_, damaris_error_string(dam_err_) ));
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
"damaris_write(PRESSURE, ...), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) ));
}
dam_err_ = damaris_end_iteration();
if (dam_err_ != DAMARIS_OK) {
OpmLog::error(fmt::format("ERORR: damariswriter::writeOutput() : ( rank:{}) damaris_end_iteration(), Damaris Error: {} ", rank_, damaris_error_string(dam_err_) ));
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
"damaris_end_iteration(), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) ));
}
}
} // end of ! isSubstep
@ -236,18 +333,24 @@ private:
}
if (dam_err_ != DAMARIS_OK) {
OpmLog::error(fmt::format("ERORR: damariswriter::writeOutput() :"
"( rank:{}) damaris_write(GLOBAL_CELL_INDEX, ...), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) ));
OpmLog::error(fmt::format("damariswriter::writeOutput() :"
"( rank:{}) damaris_write(GLOBAL_CELL_INDEX, ...), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) ));
}
std::vector<int> mpiRank(this->numElements_, rank_ ) ;
dam_err_ = damaris_write("MPI_RANK", mpiRank.data() ) ;
if (dam_err_ != DAMARIS_OK) {
OpmLog::error(fmt::format("ERORR: damariswriter::writeOutput() :"
" ( rank:{}) damaris_write(MPI_RANK, ...), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) ));
// This is an example of writing to the Damaris shared memory directly (i.e. not using
// damaris_write() to copy data there)
// We will add the MPI rank value directly into shared memory using the DamarisVar
// wrapper of the C based Damaris API.
// The shared memory is given back to Damaris when the DamarisVarInt goes out of scope.
DamarisVarInt mpi_rank_var_test(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_);
mpi_rank_var_test.setDamarisParameterAndShmem( {this->numElements_ } ) ;
// Fill the created memory area
for (int i = 0 ; i < this->numElements_; i++ )
{
mpi_rank_var_test.data()[i] = rank_ ; // write the rank vaue to the shared memory area.
}
}
void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid, std::vector<unsigned long long>& elements_rank_offsets)
@ -280,17 +383,25 @@ private:
// ToDo: Do we need to check that local ranks are 0 based ?
int temp_int = static_cast<int>(elements_rank_sizes[rank_]);
dam_err_ = damaris_parameter_set("n_elements_local", &temp_int, sizeof(int));
if (dam_err_ != DAMARIS_OK && rank_ == 0) {
OpmLog::error("Damaris library produced an error result for "
if (dam_err_ != DAMARIS_OK) {
OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
"damaris_parameter_set(\"n_elements_local\", &temp_int, sizeof(int));");
}
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that n_elements_global_max will fit in a C int type (INT_MAX)
temp_int = static_cast<int>(n_elements_global_max);
dam_err_ = damaris_parameter_set("n_elements_total", &temp_int, sizeof(int));
if (dam_err_ != DAMARIS_OK && rank_ == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_parameter_set(\"n_elements_total\", &temp_int, sizeof(int));");
if( n_elements_global_max <= std::numeric_limits<int>::max() ) {
temp_int = static_cast<int>(n_elements_global_max);
dam_err_ = damaris_parameter_set("n_elements_total", &temp_int, sizeof(int));
if (dam_err_ != DAMARIS_OK) {
OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
"damaris_parameter_set(\"n_elements_total\", &temp_int, sizeof(int));");
}
} else {
OpmLog::error(fmt::format("( rank:{} ) The size of the global array ({}) is"
"greater than what a Damaris paramater type supports ({}). ",
rank_, n_elements_global_max, std::numeric_limits<int>::max() ));
// assert( n_elements_global_max <= std::numeric_limits<int>::max() ) ;
OPM_THROW(std::runtime_error, "setupDamarisWritingPars() n_elements_global_max > std::numeric_limits<int>::max() " + std::to_string(dam_err_));
}
// Use damaris_set_position to set the offset in the global size of the array.
@ -298,19 +409,101 @@ private:
int64_t temp_int64_t[1];
temp_int64_t[0] = static_cast<int64_t>(elements_rank_offsets[rank_]);
dam_err_ = damaris_set_position("PRESSURE", temp_int64_t);
if (dam_err_ != DAMARIS_OK && rank_ == 0) {
OpmLog::error("Damaris library produced an error result for "
if (dam_err_ != DAMARIS_OK) {
OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
"damaris_set_position(\"PRESSURE\", temp_int64_t);");
}
dam_err_ = damaris_set_position("GLOBAL_CELL_INDEX", temp_int64_t);
if (dam_err_ != DAMARIS_OK && rank_ == 0) {
OpmLog::error("Damaris library produced an error result for "
if (dam_err_ != DAMARIS_OK) {
OpmLog::error("( rank:" + std::to_string(rank_)+") Damaris library produced an error result for "
"damaris_set_position(\"GLOBAL_CELL_INDEX\", temp_int64_t);");
}
dam_err_ = damaris_set_position("MPI_RANK", temp_int64_t);
if (dam_err_ != DAMARIS_OK && rank_ == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_set_position(\"MPI_RANK\", temp_int64_t);");
// Set the size of the MPI variable
DamarisVarInt mpi_rank_var(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_) ;
mpi_rank_var.setDamarisPosition({*temp_int64_t}) ;
}
void writeDamarisGridOutput( void )
{
const auto& gridView = simulator_.gridView();
Opm::GridDataOutput::SimMeshDataAccessor geomData(gridView, Dune::Partitions::interior) ;
try {
const bool hasPolyCells = geomData.polyhedralCellPresent() ;
if ( hasPolyCells ) {
OpmLog::error(fmt::format("ERORR: rank {} The DUNE geometry grid has polyhedral elements - These elements are currently not supported.", rank_ ));
}
// This is the template XML model for x,y,z coordinates defined in initDamarisXmlFile.cpp which is used to
// build the internally generated Damaris XML configuration file.
// <parameter name="n_coords_local" type="int" value="1" />
// <parameter name="n_coords_global" type="int" value="1" comment="only needed if we need to write to HDF5 in Collective mode"/>
// <layout name="n_coords_layout" type="double" dimensions="n_coords_local" comment="For the individual x, y and z coordinates of the mesh vertices" />
// <group name="coordset/coords/values">
// <variable name="x" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="PythonConduitTest" time-varying="false" />
// <variable name="y" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="PythonConduitTest" time-varying="false" />
// <variable name="z" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="PythonConduitTest" time-varying="false" />
// </group>
DamarisVarDbl var_x(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/x"), rank_) ;
// N.B. We have not set any position/offset values (using DamarisVar::SetDamarisPosition).
// They are not needed for mesh data as each process has a local geometric model.
// However, HDF5 collective and Dask arrays cannot be used for this data.
var_x.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
DamarisVarDbl var_y(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/y"), rank_) ;
var_y.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
DamarisVarDbl var_z(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/z"), rank_) ;
var_z.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
// Now we can use the shared memory area that Damaris has allocated and use it to write the x,y,z coordinates
if ( geomData.writeGridPoints(var_x, var_y, var_z) < 0)
DUNE_THROW(Dune::IOError, geomData.getError() );
// This is the template XML model for connectivity, offsets and types, as defined in initDamarisXmlFile.cpp which is used to
// build the internally generated Damaris XML configuration file.
// <parameter name="n_connectivity_ph" type="int" value="1" />
// <layout name="n_connections_layout_ph" type="int" dimensions="n_connectivity_ph" comment="Layout for connectivities " />
// <parameter name="n_offsets_types_ph" type="int" value="1" />
// <layout name="n_offsets_layout_ph" type="int" dimensions="n_offsets_types_ph+1" comment="Layout for the offsets_ph" />
// <layout name="n_types_layout_ph" type="char" dimensions="n_offsets_types_ph" comment="Layout for the types_ph " />
// <group name="topologies/topo/elements">
// <variable name="connectivity" layout="n_connections_layout_ph" type="scalar" visualizable="false" unit="" script="PythonConduitTest" time-varying="false" />
// <variable name="offsets" layout="n_offsets_layout_ph" type="scalar" visualizable="false" unit="" script="PythonConduitTest" time-varying="false" />
// <variable name="types" layout="n_types_layout_ph" type="scalar" visualizable="false" unit="" script="PythonConduitTest" time-varying="false" />
// </group>
DamarisVarInt var_connectivity(1, {std::string("n_connectivity_ph")}, std::string("topologies/topo/elements/connectivity"), rank_) ;
var_connectivity.setDamarisParameterAndShmem({ geomData.getNCorners()}) ;
DamarisVarInt var_offsets(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/offsets"), rank_) ;
var_offsets.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
DamarisVarChar var_types(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/types"), rank_) ;
var_types.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
// Copy the mesh data from the Durne grid
long i = 0 ;
Opm::GridDataOutput::ConnectivityVertexOrder vtkorder = Opm::GridDataOutput::VTK ;
i = geomData.writeConnectivity(var_connectivity, vtkorder) ;
if ( i != geomData.getNCorners())
DUNE_THROW(Dune::IOError, geomData.getError());
i = geomData.writeOffsetsCells(var_offsets);
if ( i != geomData.getNCells()+1)
DUNE_THROW(Dune::IOError,geomData.getError());
i = geomData.writeCellTypes(var_types) ;
if ( i != geomData.getNCells())
DUNE_THROW(Dune::IOError,geomData.getError());
}
catch (std::exception& e)
{
OpmLog::error(e.what());
}
}

66
ebos/eclproblem_properties.hh
View File

@ -393,17 +393,77 @@ struct EnableEclOutput<TypeTag,TTag::EclBaseProblem> {
static constexpr bool value = true;
};
#ifdef HAVE_DAMARIS
//! Enable the Damaris output by default
//! Disable the Damaris HDF5 output by default
template<class TypeTag>
struct EnableDamarisOutput<TypeTag, TTag::EclBaseProblem> {
static constexpr bool value = false;
};
// If Damaris is available, write specific variable output in parallel
template<class TypeTag>
struct EnableDamarisOutputCollective<TypeTag, TTag::EclBaseProblem> {
struct DamarisOutputHdfCollective<TypeTag, TTag::EclBaseProblem> {
static constexpr bool value = true;
};
// Save the reservoir model mesh data to the HDF5 file (even if field data HDF5 output is disabled)
template<class TypeTag>
struct DamarisSaveMeshToHdf<TypeTag, TTag::EclBaseProblem> {
static constexpr bool value = false;
};
// Save the simulation fields (currently only PRESSURE) variables to HDF5 file
template<class TypeTag>
struct DamarisSaveToHdf<TypeTag, TTag::EclBaseProblem> {
static constexpr bool value = true;
};
// Specify path and filename of a Python script to run on each end of iteration output
template<class TypeTag>
struct DamarisPythonScript<TypeTag, TTag::EclBaseProblem> {
static constexpr auto value = "";
};
// Specifiy a Paraview Catalyst in situ visualisation script (if Paraview is enabled in Damaris)
template<class TypeTag>
struct DamarisPythonParaviewScript<TypeTag, TTag::EclBaseProblem> {
static constexpr auto value = "";
};
// Specify a unique name for the Damaris simulation (used as prefix to HDF5 filenames)
template<class TypeTag>
struct DamarisSimName<TypeTag, TTag::EclBaseProblem> {
static constexpr auto value = "";
};
// Specify the number of Damaris cores (dc) to create (per-node). Must divide into the remaining ranks
// equally, e.g. mpirun -np 16 ... -> (if running on one node)
// The following are allowed:
// 1 dc + 15 sim ranks
// or 2 dc + 14 sim
// or 4 dc + 12 sim
// *not* 3 dc + 13 sim ranks
template<class TypeTag>
struct DamarisDedicatedCores<TypeTag, TTag::EclBaseProblem> {
static constexpr int value = 1;
};
// Specify the number of Damaris nodes to create
template<class TypeTag>
struct DamarisDedicatedNodes<TypeTag, TTag::EclBaseProblem> {
static constexpr int value = 0;
};
// Specify a name for the Damaris shared memory file (a unique name will be created by default)
template<class TypeTag>
struct DamarisSharedMemoryName<TypeTag, TTag::EclBaseProblem> {
static constexpr auto value = "" ; // default name is empty, will make unique if needed in DamarisKeywords()
};
// Specify the shared memory file size
template<class TypeTag>
struct DamarisSharedMemorySizeBytes<TypeTag, TTag::EclBaseProblem> {
static constexpr long value = 536870912; // 512 MB
};
// Specify the Damaris log level - if set to debug then log is flushed regularly
template<class TypeTag>
struct DamarisLogLevel<TypeTag, TTag::EclBaseProblem> {
static constexpr auto value = "info";
};
// Specify the dask file jason file that specifies the Dask scheduler etc.
template<class TypeTag>
struct DamarisDaskFile<TypeTag, TTag::EclBaseProblem> {
static constexpr auto value = "";
};
#endif
// If available, write the ECL output in a non-blocking manner
template<class TypeTag>

19
opm/simulators/flow/Main.cpp
View File

@ -235,23 +235,32 @@ void Main::setupVanguard()
}
#if HAVE_DAMARIS
void Main::setupDamaris(const std::string& outputDir,
const bool enableDamarisOutputCollective)
void Main::setupDamaris(const std::string& outputDir )
{
typedef Properties::TTag::FlowEarlyBird PreTypeTag;
if (!outputDir.empty()) {
ensureOutputDirExists(outputDir);
}
//const auto find_replace_map;
//const auto find_replace_map = Opm::DamarisOutput::DamarisKeywords<PreTypeTag>(EclGenericVanguard::comm(), outputDir);
std::map<std::string, std::string> find_replace_map;
find_replace_map = Opm::DamarisOutput::getDamarisKeywords<PreTypeTag>(EclGenericVanguard::comm(), outputDir);
// By default EnableDamarisOutputCollective is true so all simulation results will
// be written into one single file for each iteration using Parallel HDF5.
// It set to false, FilePerCore mode is used in Damaris, then simulation results in each
// If set to false, FilePerCore mode is used in Damaris, then simulation results in each
// node are aggregated by dedicated Damaris cores and stored to separate files per Damaris core.
// Irrespective of mode, output is written asynchronously at the end of each timestep.
// Using the ModifyModel class to set the XML file for Damaris.
DamarisOutput::initializeDamaris(EclGenericVanguard::comm(), EclGenericVanguard::comm().rank(), outputDir, enableDamarisOutputCollective);
DamarisOutput::initializeDamaris(EclGenericVanguard::comm(),
EclGenericVanguard::comm().rank(),
find_replace_map);
int is_client;
MPI_Comm new_comm;
int err = damaris_start(&is_client);
// damaris_start() is where the Damaris Server ranks will block, until damaris_stop()
// is called from the client ranks
int err = damaris_start(&is_client);
isSimulationRank_ = (is_client > 0);
if (isSimulationRank_ && err == DAMARIS_OK) {
damaris_client_comm_get(&new_comm);

17
opm/simulators/flow/Main.hpp
View File

@ -72,6 +72,10 @@
#include <opm/simulators/utils/ParallelEclipseState.hpp>
#endif
#if HAVE_DAMARIS
#include <opm/simulators/utils/DamarisKeywords.hpp>
#endif
#include <cassert>
#include <cstdlib>
#include <iostream>
@ -323,9 +327,14 @@ private:
deckFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, EclDeckFileName);
outputDir = EWOMS_GET_PARAM(PreTypeTag, std::string, OutputDir);
}
if (outputDir.empty()) {
outputDir = ".";
}
#if HAVE_DAMARIS
enableDamarisOutput_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutput);
// Reset to false as we cannot use Damaris if there is only one rank.
if ((enableDamarisOutput_ == true) && (EclGenericVanguard::comm().size() == 1)) {
std::string msg ;
@ -333,10 +342,9 @@ private:
OpmLog::warning(msg);
enableDamarisOutput_ = false ;
}
if (enableDamarisOutput_) {
this->setupDamaris(outputDir,
EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutputCollective));
this->setupDamaris(outputDir); // Damaris server ranks will block here until damaris_stop() is called by client ranks
}
#endif // HAVE_DAMARIS
@ -706,8 +714,7 @@ private:
}
#if HAVE_DAMARIS
void setupDamaris(const std::string& outputDir,
const bool enableDamarisOutputCollective);
void setupDamaris(const std::string& outputDir);
#endif
int argc_{0};

271
opm/simulators/utils/DamarisKeywords.cpp
View File

@ -1,6 +1,7 @@
/*
Copyright 2021 Equinor.
Copyright 2023 Inria.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
@ -17,55 +18,249 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#include <Damaris.h>
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <damaris/env/Environment.hpp>
#include <string>
#include <map>
#include <random>
#include <fstream>
/*
Below is the Damaris Keywords supported by Damaris to be filled
/**
Below are the Damaris Keywords supported by Damaris to be filled
in the built-in XML file.
The entries in the map below will be filled by the corresponding
Damaris Keywords. Yet, only output directory and FileMode are to
be chosen by the user
Damaris Keywords.
The command line arguments are defined in ebos/damariswriter.hh
and defaults are set in ebos/eclproblem_properties.hh
*/
namespace Opm::DamarisOutput
{
std::map<std::string, std::string>
DamarisKeywords(std::string OutputDir, bool enableDamarisOutputCollective)
bool FileExists(const std::string& filename_in,
const Parallel::Communication& comm)
{
if (enableDamarisOutputCollective) {
std::map<std::string, std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_", "536870912"},
{"_DC_REGEX_", "1"},
{"_DN_REGEX_", "0"},
{"_File_Mode", "Collective"},
{"_MORE_VARIABLES_REGEX_", ""},
{"_PATH_REGEX_", OutputDir},
{"_MYSTORE_OR_EMPTY_REGEX_", "MyStore"},
{"_PYTHON_OR_EMPTY_REGEX_", ""},
{"_PYTHON_XML_NAME_", "#"},
{"_PARAVIEWPY_OR_EMPTY_REGEX_", ""},
{"_MESHNAME_OR_HASH_", "#"},
};
return damaris_keywords;
} else {
std::map<std::string, std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_", "536870912"},
{"_DC_REGEX_", "1"},
{"_DN_REGEX_", "0"},
{"_File_Mode", "FilePerCore"},
{"_MORE_VARIABLES_REGEX_", ""},
{"_PATH_REGEX_", OutputDir},
{"_MYSTORE_OR_EMPTY_REGEX_", "MyStore"},
{"_PYTHON_OR_EMPTY_REGEX_", ""},
{"_PYTHON_XML_NAME_", "#"},
{"_PARAVIEWPY_OR_EMPTY_REGEX_", ""},
{"_MESHNAME_OR_HASH_", "#"},
};
return damaris_keywords;
// From c++17 : std::filesystem::exists(filename_in);
int retint = 0;
std::ifstream filestr;
bool file_exists = false;
if ((filename_in.length() == 0) || (filename_in == "#") ) {
return file_exists;
}
if (comm.rank() == 0) {
filestr.open(filename_in);
file_exists = true;
if(filestr.fail()) {
retint = 0;
} else {
retint = 1;
filestr.close();
}
}
comm.broadcast(&retint, 1, 0);
if (retint == 1) {
file_exists = true;
}
return (file_exists);
}
} // namespace Opm::DamarisOutput
void DamarisSettings::SetRandString(void)
{
// rand_value_str_ = damaris::Environment::GetMagicNumber(comm); // requires Damaris >= v1.9.2
// We will create a random value.
// Seed with a real random value, if available
std::random_device r;
// Choose a random number between 0 and MAX_INT
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, std::numeric_limits<int>::max());
int rand_int = uniform_dist(e1);
rand_value_str_ = std::to_string(rand_int) ;
}
std::map<std::string, std::string>
DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& comm,
const std::string& OutputDir)
{
SetRandString() ; // sets rand_value_str_ used for naming things that might need a unique name
std::string saveToHDF5_str("MyStore");
if (! saveToDamarisHDF5_ ){
saveToHDF5_str = "#";
}
// These strings are used to comment out an XML element if it is not reqired
std::string disablePythonXMLstart("!--");
std::string disablePythonXMLfin("--");
std::string disableParaviewXMLstart("!--");
std::string disableParaviewXMLfin("--");
std::string publishToPython_str("#"); // to be changed to the name of the PyScript XML element
#ifdef HAVE_PYTHON_ENABLED
// Test if input Python file exists and set the name of the script for <variable ... script="" > )XML elements
if (pythonFilename_ != ""){
if (FileExists(pythonFilename_, comm)) {
publishToPython_str="PythonScript"; // the name of the PyScript XML element
disablePythonXMLstart.clear();
disablePythonXMLfin.clear();
} else {
pythonFilename_.clear(); // set to empty if it does not exist
disablePythonXMLstart = std::string("!--");
disablePythonXMLfin = std::string("--");
}
}
#else
OpmLog::info(fmt::format("INFO: Opm::DamarisOutput::DamarisKeywords() : Python is not enabled in the Damaris library. "
"The commandline --damaris-python-script={} will be set to empty string", pythonFilename_));
pythonFilename_.clear();
#endif
#ifdef HAVE_PARAVIEW_ENABLED
// Test if input Paraview Python file exists
if (paraviewPythonFilename_ != ""){
if (FileExists(paraviewPythonFilename_, comm)) {
disableParaviewXMLstart.clear();
disableParaviewXMLfin.clear();
} else {
paraviewPythonFilename_.clear(); // set to empty if it does not exist
disableParaviewXMLstart = std::string("!--");
disableParaviewXMLfin = std::string("--");
}
}
#else
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : Paraview is not enabled in the Damaris library. "
"The commandline --damaris-python-paraview-script={} will be set to empty string",
paraviewPythonFilename_));
paraviewPythonFilename_.clear();
#endif
// Flag error if both scripts are enabled
if ((pythonFilename_.size() > 0) && (paraviewPythonFilename_.size() > 0) )
{
// A work around of this issue is to remove the Paraview mpi4py library (use print(inspect.getfile(mpi4py)))
// and then possibly not use mpi4py in the Paraview script code. OR try to install paraview mpi4py with headers.
OPM_THROW(std::runtime_error, "ERROR: Both the Python (--damaris-python-script command line argument) and Paraview Python "
"(--damaris-python-paraview-script command line argument) scripts are valid, however only one "
"type of analysis is supported in a single simulation (due to Paraview installing mpi4py library "
"locally and without header files). "
"Please choose one or the other method of analysis for now. Exiting." );
}
std::string saveMeshToHDF5_str("#");
if (saveMeshToHDF5_ == true) {
enableDamarisOutputCollective_ = false ;
saveMeshToHDF5_str = "MyStore" ;
}
std::string damarisOutputCollective_str;
if (enableDamarisOutputCollective_) {
damarisOutputCollective_str = "Collective";
} else {
damarisOutputCollective_str = "FilePerCore";
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <option key=\"FileMode\"> {} </option> ",
damarisOutputCollective_str));
std::string simName_str;
// Check if simulation name was given on command line
// The simulation name is used as a prefix to name HDF5 files
if (damarisSimName_.empty()) {
simName_str = "opm-flow-" + rand_value_str_;
} else {
simName_str = damarisSimName_;
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <simulation name={} ",
simName_str));
// A different shared memory buffer name is important if multiple simulations
// are running on the same node, as one simulation will remove the buffer when it exits,
// which will remove the buffer for other simulations.
std::string shmemName_str;
if ( shmemName_.empty()) {
shmemName_str = "opm-damaris-" + rand_value_str_;
} else {
shmemName_str = shmemName_ ;
}
std::string shmemSizeBytes_str;
if (shmemSizeBytes_ != 0) {
shmemSizeBytes_str = std::to_string(shmemSizeBytes_);
} else {
shmemSizeBytes_str = "536870912"; // 512 MB
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <buffer name={} size={} ",
shmemName_str, shmemSizeBytes_str));
if ((nDamarisCores_ > 0) && (nDamarisNodes_ > 0))
{
nDamarisNodes_ = 0; // Default is to use Damaris Cores
}
std::string nDamarisCores_str;
if ( nDamarisCores_ != 0 ) {
nDamarisCores_str = std::to_string(nDamarisCores_);
} else {
nDamarisCores_str = "0";
}
std::string nDamarisNodes_str;
if ( nDamarisNodes_ != 0 ) {
nDamarisNodes_str = std::to_string(nDamarisNodes_);
} else {
nDamarisNodes_str = "0";
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <dedicated cores={} nodes={} ",
nDamarisCores_str, nDamarisNodes_str));
std::string logLevel_str(damarisLogLevel_);
std::string logFlush_str("false");
if ((logLevel_str == "debug") || (logLevel_str == "trace") ) {
logFlush_str = "true";
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <log FileName={}/damaris_log/{} Flush={} LogLevel={} ",
OutputDir, simName_str, logFlush_str, logLevel_str));
std::map<std::string, std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_", shmemSizeBytes_str},
{"_DC_REGEX_", nDamarisCores_str},
{"_DN_REGEX_", nDamarisNodes_str},
{"_File_Mode", damarisOutputCollective_str},
{"_MORE_VARIABLES_REGEX_", ""},
{"_PATH_REGEX_", OutputDir}, /* Do Not change the string "_PATH_REGEX_" as it is used to search for the output path */
{"_MYSTORE_OR_EMPTY_REGEX_", saveToHDF5_str},
{"_MYSTORE_MESH_OR_EMPTY_REGEX_", saveMeshToHDF5_str},
{"_PARAVIEW_PYTHON_SCRIPT_",paraviewPythonFilename_}, /* this has to be before _PYTHON_SCRIPT_ entry */
{"_PYTHON_SCRIPT_",pythonFilename_}, /* if a Python script is specified then assume that we want to publish the data to Python */
{"_PRESSURE_UNIT_","Pa"},
{"_MAKE_AVAILABLE_IN_PYTHON_",publishToPython_str}, /* must match <pyscript name="PythonScript" */
{"_SIM_NAME_",simName_str},
{"_SHMEM_NAME_",shmemName_str},
{"_LOG_LEVEL_",logLevel_str},
{"_LOG_FLUSH_",logFlush_str},
{"_DISABLEPYTHONSTART_",disablePythonXMLstart},
{"_DISABLEPYTHONFIN_",disablePythonXMLfin},
{"_DISABLEPARAVIEWSTART_",disableParaviewXMLstart},
{"_DISABLEPARAVIEWFIN_",disableParaviewXMLfin},
{"_DASK_SCHEDULER_FILE_",damarisDaskFile_},
};
return damaris_keywords;
}
} // namespace Opm::DamarisOutput

72
opm/simulators/utils/DamarisKeywords.hpp
View File

@ -23,6 +23,10 @@
#include <string>
#include <map>
#include <opm/simulators/utils/ParallelCommunication.hpp>
#include <ebos/damariswriter.hh>
/*
Below is the std::map with the keywords that are supported by Damaris.
@ -33,10 +37,74 @@
namespace Opm::DamarisOutput
{
/**
* Returns true if the file exists.
* Tests to see if filename string is empty
* or the "#" character and if so returns false.
* Tests for file existance on rank 0 and
* passes result via MPI to all other ranks.
*/
bool FileExists(const std::string& filename_in,
const Parallel::Communication& comm);
std::map<std::string,std::string> DamarisKeywords(std::string outputDir, bool enableDamarisOutputCollective);
struct DamarisSettings {
bool enableDamarisOutputCollective_ = true;
bool saveToDamarisHDF5_ = true;
// if saveMeshToDamarisHDF5 is true, requires enableDamarisOutputCollective to be false
// (until offsets are are added to mesh data for collective writing)
bool saveMeshToHDF5_ = false;
std::string pythonFilename_;
std::string paraviewPythonFilename_;
std::string damarisSimName_; // empty and set to "opm-flow-<random-number>" if none provided on command line. Used as prefix to HDF5 filenames
std::string shmemName_; // empty and needs to be unique if multiple simulations are running on the same server/node. Used to name the Damaris shared memory region.
std::string damarisLogLevel_ = "info";
std::string damarisDaskFile_ = "";
int nDamarisCores_ = 1; // this is the number of (Damaris server) cores per node
int nDamarisNodes_ = 0;
long shmemSizeBytes_ = 536870912; // 512 MB
std::string rand_value_str_ ; // to be added to sheared memory name to make unique
std::map<std::string, std::string>
getKeywords(const Parallel::Communication& comm,
const std::string& OutputDir);
void SetRandString(void); // sets the value of rand_value_str_
};
/**
* Creates the map of search strings and repacement strings that will be used to
* modify a templated Damaris XML file which will be used to intialize Damaris.
* This function will access all the OPM flow comand line arguments related to
* Damaris and perform checks and logic so as to create a valid XML file.
* N.B. The created XML file can be overridden using an environment variable
* FLOW_DAMARIS_XML_FILE that points to a Damaris XML file.
*/
template<class TypeTag>
std::map<std::string, std::string>
getDamarisKeywords(const Parallel::Communication& comm, const std::string& OutputDir)
{
DamarisSettings settings;
// Get all of the Damaris keywords (except for --enable-damaris, which is used in simulators/flow/Main.hpp)
// These command line arguments are defined in ebos/damariswriter.hh and defaults are set in ebos/eclproblem_properties.hh
settings.enableDamarisOutputCollective_ = EWOMS_GET_PARAM(TypeTag, bool, DamarisOutputHdfCollective);
settings.saveMeshToHDF5_ = EWOMS_GET_PARAM(TypeTag, bool, DamarisSaveMeshToHdf);
settings.saveToDamarisHDF5_ = EWOMS_GET_PARAM(TypeTag, bool, DamarisSaveToHdf);
settings.pythonFilename_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisPythonScript);
settings.paraviewPythonFilename_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisPythonParaviewScript);
settings.damarisSimName_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisSimName);
settings.nDamarisCores_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedCores);
settings.nDamarisNodes_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedNodes);
settings.shmemSizeBytes_ = EWOMS_GET_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes);
settings.shmemName_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisSharedMemoryName);
settings.damarisLogLevel_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisLogLevel);
settings.damarisDaskFile_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisDaskFile);
return settings.getKeywords(comm, OutputDir);
}
} // namespace Opm::DamarisOutput
#endif
#endif

70
opm/simulators/utils/DamarisOutputModule.cpp
View File

@ -18,10 +18,11 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#define XSD_CXX11_TEMPLATE_ALIAS 1
#include <config.h>
#include <damaris/model/ModifyModel.hpp>
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <opm/simulators/utils/DamarisOutputModule.hpp>
@ -30,48 +31,55 @@
#include <fmt/format.h>
namespace Opm::DamarisOutput
{
std::string initDamarisXmlFile(); // Defined in initDamarisXMLFile.cpp, to avoid messing up this file.
/**
* Initialize Damaris by either reading a file specified by the environment variable FLOW_DAMARIS_XML_FILE or
* by filling in the XML file and storing it in the chosen directory
*/
void
initializeDamaris(MPI_Comm comm, int mpiRank, std::string outputDir, bool enableDamarisOutputCollective)
initializeDamaris(const MPI_Comm comm, const int mpiRank, const std::map<std::string, std::string>& find_replace_map )
{
if (outputDir.empty()) {
outputDir = ".";
}
// Prepare the XML file
std::string damaris_config_xml = initDamarisXmlFile();
damaris::model::ModifyModel myMod = damaris::model::ModifyModel(damaris_config_xml);
// The map will make it precise the output directory and FileMode (either FilePerCore or Collective storage)
// The map file find all occurences of the string in position 1 and repalce it/them with string in position 2
std::map<std::string, std::string> find_replace_map = DamarisKeywords(outputDir, enableDamarisOutputCollective);
myMod.RepalceWithRegEx(find_replace_map);
std::string damaris_xml_filename_str = outputDir + "/damaris_config.xml";
if (mpiRank == 0) {
myMod.SaveXMLStringToFile(damaris_xml_filename_str);
}
int damaris_err;
int dam_err;
/* Get the name of the Damaris input file from an environment variable if available */
const char* cs_damaris_xml_file = getenv("FLOW_DAMARIS_XML_FILE");
if (cs_damaris_xml_file != NULL) {
std::cout << "INFO: initializing Damaris from environment variable FLOW_DAMARIS_XML_FILE: "
<< cs_damaris_xml_file << std::endl;
damaris_err = damaris_initialize(cs_damaris_xml_file, MPI_COMM_WORLD);
if (damaris_err != DAMARIS_OK) {
std::cerr << "ERROR: damaris_initialize() error via FLOW_DAMARIS_XML_FILE=" << cs_damaris_xml_file
<< std::endl;
if (cs_damaris_xml_file != NULL)
{
OpmLog::info(std::string("Initializing Damaris from environment variable FLOW_DAMARIS_XML_FILE: ") + cs_damaris_xml_file);
dam_err = damaris_initialize(cs_damaris_xml_file, comm);
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("ERORR: damariswriter::initializeDamaris() : ( rank:{}) "
"damaris_initialize({}, comm), Damaris Error: {} ",
mpiRank, cs_damaris_xml_file, damaris_error_string(dam_err) ));
}
} else {
std::cout << "INFO: initializing Damaris using internally built file:" << damaris_xml_filename_str << std::endl;
damaris_err = damaris_initialize(damaris_xml_filename_str.c_str(), comm);
if (damaris_err != DAMARIS_OK) {
std::cerr << "ERROR: damaris_initialize() error via built file:" << std::endl << myMod.GetConfigString();
// Prepare the inbuilt XML file
std::string damaris_config_xml = initDamarisXmlFile(); // This is the template for a Damaris XML file
damaris::model::ModifyModel myMod = damaris::model::ModifyModel(damaris_config_xml);
// The map file find all occurences of the string in position 1 and replace it/them with string in position 2
// std::map<std::string, std::string> find_replace_map = DamarisKeywords(outputDir, enableDamarisOutputCollective);
myMod.RepalceWithRegEx(find_replace_map);
std::string outputDir = find_replace_map.at("_PATH_REGEX_");
std::string damaris_xml_filename_str = outputDir + "/damaris_config.xml";
if (mpiRank == 0) {
myMod.SaveXMLStringToFile(damaris_xml_filename_str);
}
OpmLog::info("Initializing Damaris using internally built file: " + damaris_xml_filename_str +
" (N.B. use environment variable FLOW_DAMARIS_XML_FILE to override)");
dam_err = damaris_initialize(damaris_xml_filename_str.c_str(), comm);
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::initializeDamaris() : ( rank:{}) "
"damaris_initialize({}, comm), Damaris Error: {}. Error via OPM internally built file:",
mpiRank, cs_damaris_xml_file, damaris_error_string(dam_err) ));
}
}
}

17
opm/simulators/utils/DamarisOutputModule.hpp
View File

@ -18,7 +18,14 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <opm/common/OpmLog/OpmLog.hpp>
#include <limits>
#include <stdexcept>
#include <string>
#include <fmt/format.h>
#include <Damaris.h>
#include <opm/simulators/utils/ParallelCommunication.hpp>
@ -34,7 +41,13 @@ namespace Opm::DamarisOutput
{
// Initialize an XML file
std::string initDamarisXmlFile();
// Initialize Damaris by filling in th XML file and stroring it in the chosed directory
void initializeDamaris(MPI_Comm comm, int mpiRank, std::string OutputDir, bool enableDamarisOutputCollective);
/**
* Initialize Damaris by either:
* 1/ Filling in a templated XML file and storing it in the chosen directory (output directory)
* 2/ Reading a file specified by the environment variable FLOW_DAMARIS_XML_FILE
*
*/
void initializeDamaris(const MPI_Comm comm, const int mpiRank, const std::map<std::string, std::string>& find_replace_map );
} // namespace Opm::DamarisOutput

691
opm/simulators/utils/DamarisVar.hpp Normal file
View File

@ -0,0 +1,691 @@
/*
Copyright 2023 Inria, Bretagne–Atlantique Research Center
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef DAMARISVAR_HPP
#define DAMARISVAR_HPP
#include <cassert>
#include <iostream>
#include <sstream>
#include <string>
#include <typeinfo>
#define BOOST_BIND_GLOBAL_PLACEHOLDERS 1
#include <Damaris.h>
/*
File: DamarisVar.hpp
Author: Joshua Bowden, Inria
Date: 06/02/2023
The DamarisVar class can be used to allocate memory in the Damaris shared
memory region and a user can supply the data required for the variable via the data_ptr_
or data() method. The data will then be directly available on the Damaris server side
(server cores) plugin code. The templated type used should match the one specified
in the Damaris XML file for the variable name, if not an error is raised.
*/
namespace Opm
{
namespace DamarisOutput
{
/**
* This class contains the extra elements that need to be part of a Damaris
* <variable> type. They are simple string values that may reference other XML
* elements (and could be checked for existence etc.)
*/
class DamarisVarXMLAttributes
{
std::string layout_; //!< Reference string to the XML attribute layout being
//!< used to describe the shape of the variable. This is
//!< a required attribute.
std::string mesh_; //!< Reference string to the XML attribute mesh element - the mesh
//!< is used to define the spatial layout of data and is used by
//!< visualization backends to generate 2D/3D model images
std::string type_; //!< Reference string to the XML attribute type of data - "scalar"
//!< or "vector" (others tensor maybe). TODO: check if this
//!< attribute is used by the Damaris library anywhere.
std::string visualizable_; //!< Reference string to the XML attribute property that
//!< data can be sent to vis backends - "true" | "false"
std::string unit_; //!< Reference string to the XML attribute element denoting
//!< unit of the data
std::string time_varying_; //!< Reference string to the XML attribute to indicate if
//!< data changes over iterations - "true" | "false"
std::string centering_; //!< Reference string to the XML attribute to indicate
//!< where data aligns on a mesh - "zonal" | "nodal"
std::string store_; //!< Reference string to the XML attribute to indicate if data
//!< should be passed to I/O store (e.g. to HDF5 plugin)
std::string script_; //!< Reference string to the XML attribute to indicate if
//!< data should be published as Python NumPy data
std::string select_mem_; //!< Reference string to the XML attribute select. The
//!< referenced variables data is used as indices to select
//!< dat from memory to reorder output in the collective HDF5
//!< data writer (Damaris version 1.8+)
std::string select_file_; //!< Reference string to the XML attribute select. The
//!< referenced variables data is used as indices to select
//!< positions in the dataset file to reorder output in the
//!< collective HDF5 data writer (Damaris version 1.8+)
std::string select_subset_; //!< Reference string to the XML attribute select. Used to
//!< specify the output dataset shape and how much data
//!< each rank contributes to it and the global offsets to
//!< the ranks data (Damaris version 1.8+)
public:
DamarisVarXMLAttributes()
{
// Additional data needed to complete an XML <variable> element
layout_ = "";
mesh_ = "";
type_ = "scalar"; // This is probably not needed as vector data is defined using
// the Layout paramter. Could be useful for cross checking
visualizable_ = "false";
unit_ = "";
time_varying_ = "true";
centering_ = "zonal";
store_ = "";
script_ = "";
select_mem_ = "";
}
/**
* Creates the XML representation of the variable from the available strings
*/
std::string ReturnXMLForVariable(void)
{
std::ostringstream var_sstr;
var_sstr << " layout=\"" << this->layout_ << "\"";
if (this->mesh_ != "")
var_sstr << " mesh=\"" << this->mesh_ << "\"";
if (this->type_ != "")
var_sstr << " type=\"" << this->type_ << "\"";
if (this->visualizable_ != "")
var_sstr << " visualizable=\"" << this->visualizable_ << "\"";
if (this->unit_ != "")
var_sstr << " unit=\"" << this->unit_ << "\"";
if (this->time_varying_ != "")
var_sstr << " time_varying=\"" << this->time_varying_ << "\"";
if (this->centering_ != "")
var_sstr << " centering=\"" << this->centering_ << "\"";
if (this->store_ != "")
var_sstr << " store=\"" << this->store_ << "\"";
if (this->script_ != "")
var_sstr << " script=\"" << this->script_ << "\"";
if (this->select_mem_ != "")
var_sstr << " select-mem=\"" << this->select_mem_ << "\"";
if (this->select_file_ != "")
var_sstr << " select-file=\"" << this->select_file_ << "\"";
if (this->select_subset_ != "")
var_sstr << " select-subset=\"" << this->select_subset_ << "\"";
return (var_sstr.str());
}
};
class DamarisVarBase
{
public:
virtual ~DamarisVarBase(void) {};
virtual void printError(void) = 0;
virtual bool hasError(void) = 0;
virtual void setDamarisParameterAndShmem(const std::vector<int>& paramSizeVal) = 0;
virtual void setDamarisParameter(const std::vector<int>& paramSizeVal) = 0;
virtual void setDamarisPosition(const std::vector<int64_t>& positionsVals) = 0;
virtual void setPointersToDamarisShmem(void) = 0;
virtual void commitVariableDamarisShmem(void) = 0;
virtual void clearVariableDamarisShmem(void) = 0;
virtual std::string& variable_name(void) = 0;
}; // class DamarisVarBase
/**
* class to store a Damaris variable representation for the XML file
* (can be used with /ref class DamarisKeywords).
*
* It is thought that the details stored in the object can be used to pass
* into an XML generation function e.g. DamarisKeywords
*
*/
template <typename T>
class DamarisVar : public DamarisVarBase
{
int num_params_; //!< Each paramater name string will need a value and they
//!< are set in SetDamarisParameter()
std::vector<int> param_sizes_; //!< The value for any paramaters that are being used to
//!< describe the size of the variables data array
std::vector<int64_t> positions_; //!< The offsets into the array that the data in the Variable
//!< starts from for this rank.
int rank_; //!< Rank of process - used for error reporting.
bool parameters_set_; //!< set to true after SetDamarisParameter() is call to
//!< ensure the variable has correct size for memory
//!< allocation in SetPointersToDamarisShmem()
std::vector<std::string> param_names_; //!< Contains one paramater name for each paramater that a
//!< variable depends on (via it's Layout)
std::string variable_name_; //!< Reference string to the XML attribute name of
//!< the variable.
int dam_err_; //!< Set to != DAMARIS_OK if a Damaris error was returned by a
//!< Damaris API function call
bool has_error_;
std::ostringstream dam_err_sstr_; //!< Use dam_err_sstr.str() to return an
//!< error string describing detected error
DamarisVarXMLAttributes xml_attributes_; //!< The extra elements that need to be part of a Damaris
//!< <variable> type. They are simple string values that
//!< may reference other XML elements (and could be
//!< checked for existence etc.)
T* data_ptr_; //!< This pointer will be mapped to the Damaris shared memory
//!< area for the variable in the SetPointersToDamarisShmem()
//!< method. The type T will match the Layout type
size_t current_size_; //!< The total number of elements that may be held by this
//!< part of the variable - returned by the size() method.
//!< N.B. the actual size of the data area is dependent on
//!< how the <variable> XML is written, as paramaters can
//!< be augmented by basic maths relationships. This value
//!< may not even be initialised if ParameterIsSet() method
//!< is being used (e.g. in version 2/ of the constructor below).
public:
/**
* Constructor - sets private data values and dos not initialise the shared memory area.
*
* N.B. These objects need a matching <variable ...> and <paramater ...> entries in the Damaris XML file
*
* Two usages:
* Example XML definition:
* <parameter name="my_param_name1" type="int" value="1" />
* <parameter name="my_param_name2" type="int" value="1" />
* <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" />
* <variable name="MYVARNAME" layout="my_layout" visualizable="true"/>
*
* 1/ The variable's layout needs to be initialised via parameters :
* // Create the DamarisVar object:
* damaris::model::DamarisVar<int> MYVARNAME_2d(2,{std::string("my_param_name1"),
* std::string("my_param_name2")},
* {100, 25},
* std::string("MYVARNAME"), rank_);
* // sets the paramater sizes (so, here, my_param_name1 == 25 and my_param_name2 == 100)
* MYVARNAME_2d.SetDamarisParameterAndShmem( {25, 100 } };
* // Get a pointer to the memeory and use it
* T * mymemory = MYVARNAME_2d.data();
* ... write data to mymemory ....
* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
* or,
* 2/ The variable's layout has been initialised via parameters in another variable
* (i.e. "my_param_name1" and "my_param_name2" have been previously set in the code)
* // Create the DamarisVar object:
* damaris::model::DamarisVar<int> MYVARNAME_2d(2, {std::string("my_param_name1"),
std::string("my_param_name2")},
* std::string("MYVARNAME"), rank_);
*
* // explicitly state that the paramater values have been set somewhere else in the code previously.
* MYVARNAME_2d.ParameterIsSet();
*
* N.B. This will not set the internal current_size_ value so the size() value will
* not be correct <- This is important to remember
*
* MYVARNAME_2d.SetPointersToDamarisShmem()
* // Get a pointer to the memeory and use it
* T * mymemory = MYVARNAME_2d.data();
* ... write data to mymemory ....
* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
*
* /param [IN] dims Used to check that the inputs to SetDamarisPosition()
* have the same number of values - one value for each dimension
* /param [IN] param_names The name the Damaris paramaters. These names (in typical use) control
* a Damaris variables size (names are defined in the Damaris XML file).
* /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file)
* /param [IN] rank The rank of the process. Used for error output.
*/
DamarisVar(int dims, std::vector<std::string> param_names, std::string variable_name, int rank)
: param_names_(param_names)
, variable_name_(variable_name)
, rank_(rank)
{
dam_err_ = DAMARIS_OK;
assert(param_names_.size() == dims);
assert(dims > 0);
has_error_ = false;
// Check that our template type T matches out Damaris XML <layout> type
TestType(variable_name);
if (hasError()) {
printError(); // throws a runtime error, with error message from
// dam_err_sstr_
}
current_size_ = 0;
num_params_ = param_names_.size();
param_sizes_.resize(num_params_);
positions_.resize(dims);
data_ptr_ = nullptr;
parameters_set_ = false;
has_error_ = false;
}
/**
* Constructor - Sets private data values and also initialises the Damaris shared memory area for writing (and
* reading) by specifying the values for the variables parameters . i.e. makes the data() pointer available and
* sets the size of the memory block it points to.
*
* N.B. These objects need a matching <variable ...> and <paramater ...> entries in the Damaris XML file
*
* Example use:
* Example XML definition:
* <parameter name="my_param_name1" type="int" value="1" />
* <parameter name="my_param_name2" type="int" value="1" />
* <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" comment="This
* is a 2D variable" /> <variable name="MYVARNAME" layout="my_layout" visualizable="true"/>
* // The paramaters are intialized in the constructor code
* damaris::model::DamarisVar<int> MYVARNAME_2d(2,{std::string("my_param_name1"),
* std::string("my_param_name2")}, {100, 25}, std::string("MYVARNAME"), rank_); T * mymemory =
* MYVARNAME_2d.data();
* ... write data to mymemory ....
* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
*
* /param [IN] dims Used to check that the inputs to SetDamarisPosition() have
* the same number of values - one value for each dimension
* /param [IN] param_names The name the Damaris paramaters. These names (in typical use)
* control a Damaris variables size (names are defined in the Damaris XML file).
* /param [IN] param_values The values of the paramaters - this defines how much memory we will
* have access to in the shared memory area (on the current and ongoing iterations,
* until later modified to new values)
* /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file)
* /param [IN] rank The rank of the process. Used for error output.
*/
DamarisVar(int dims,
std::vector<std::string> param_names,
std::vector<int> param_values,
std::string variable_name,
int rank)
: param_names_(param_names)
, variable_name_(variable_name)
, rank_(rank)
{
DamarisVar(dims, param_names, variable_name, rank);
setDamarisParameterAndShmem(param_values); // Initialise the memory size in the constructor.
}
~DamarisVar(void)
{
if (data_ptr_ != nullptr) {
commitVariableDamarisShmem();
clearVariableDamarisShmem();
}
if (this->hasError())
printError(); // flush out any error messages
}
/**
* Method to check that the template paramater T is the same as the requested
* type for the variable in the XML file
*/
bool TestType(std::string variable_name)
{
bool resbool = true;
// This gets the type of the Damaris XML <variable>'s <layout>
DAMARIS_TYPE_STR vartype;
dam_err_ = damaris_get_type(variable_name.c_str(), &vartype);
if (dam_err_ != DAMARIS_OK) {
dam_err_sstr_ << " ERROR rankDamarisVar::DamarisVar () damaris_get_type(\"" << variable_name_
<< "\", vartype); Damaris error = " << damaris_error_string(dam_err_) << std::endl;
has_error_ = true;
return (false);
}
T test_id;
const std::type_info& t1 = typeid(test_id);
if (vartype == DAMARIS_TYPE_DOUBLE) {
double td = 0.0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_FLOAT) {
float td = 0.0f;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_CHAR) {
char td = 0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_UCHAR) {
unsigned char td = 0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_SHORT) {
short td = 0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_USHORT) {
unsigned short td = 0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_INT) {
int td = 0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_UINT) {
unsigned int td = 0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_LONG) {
long td = 0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_ULONG) {
unsigned long td = 0;
const std::type_info& t2 = typeid(td);
if (t1 != t2) {
formatTypeError(variable_name, t1.name(), t2.name());
resbool = false;
}
} else if (vartype == DAMARIS_TYPE_UNDEFINED) {
dam_err_sstr_ << " ERROR rank =" << rank_ << " : DamarisVar::DamarisVar():: \"" << variable_name
<< "\" has type DAMARIS_TYPE_UNDEFINED" << std::endl;
has_error_ = true;
resbool = false;
} else {
dam_err_sstr_ << " ERROR rank =" << rank_ << " : DamarisVar::DamarisVar():: \"" << variable_name
<< "\" is not of available type " << std::endl;
has_error_ = true;
resbool = false;
}
return resbool;
}
/**
* Allow a user to indicate that the Damaris variable has allocated a size -
* This method is usefull as a single parameter can control one or more
* layouts and a single layout can describe the size of multiple <variable>
* elements. i.e. Use when the current variable has had it's paramater(s) set
* through via another variable.
*/
void parameterIsSet()
{
parameters_set_ = true;
}
void printError(void)
{
OPM_THROW(std::runtime_error, dam_err_sstr_.str());
}
bool hasError(void)
{
return (has_error_);
}
/**
* Returns the data pointer to shared memory, or nullptr if it has not been
* allocated
*/
T* data(void)
{
if (parameters_set_ == true) {
return (data_ptr_); // This still could be nullptr
} else {
return (nullptr);
}
}
std::string& variable_name(void)
{
return (variable_name_);
}
/**
* Creates the XML representation of the variable from the available strings
*/
std::string returnXMLForVariable(void)
{
std::ostringstream var_sstr;
var_sstr << "<variable "
<< " name=\"" << variable_name_ << "\"";
var_sstr << xml_attributes_.ReturnXMLForVariable();
var_sstr << " /> ";
return var_sstr.str();
}
/**
* Method to set the Damaris paramater values and set the shmem region \ref
* data_ptr_
*
* /param [IN] paramSizeVal : A vector of values to set the Damaris paramters
* to. One element per param_names_ string
*
*
*/
void setDamarisParameterAndShmem(const std::vector<int>& paramSizeVal)
{
this->setDamarisParameter(paramSizeVal);
this->setPointersToDamarisShmem();
}
/**
* Returns the number of elements in the memory area.
* Used as a method for compatibility with std::vector
*/
size_t size()
{
if (parameters_set_ == true) {
return current_size_;
} else {
return 0;
}
}
/**
* Method to set the Damaris paramater values. Also calculates the total
* number of elements in the variable (current_size_) that is returned bt
* size() method.
*
* /param [IN] paramSizeVal : An pointer to a value or array of values to
* set. One element per param_names_ string
*
* /implicit : Implicitly uses the array of paramater names:
* \ref param_names_
*/
void setDamarisParameter(const std::vector<int>& paramSizeVal)
{
assert(paramSizeVal.size() == num_params_);
bool resbool = true;
size_t total_size = 1;
for (int varnum = 0; varnum < num_params_; varnum++) {
param_sizes_[varnum] = paramSizeVal[varnum];
total_size *= param_sizes_[varnum];
dam_err_ = damaris_parameter_set(param_names_[varnum].c_str(), &paramSizeVal[varnum], sizeof(int));
if (dam_err_ != DAMARIS_OK) {
dam_err_sstr_ << " ERROR rank =" << rank_ << " : class DamarisVar : damaris_parameter_set(\""
<< param_names_[varnum] << "\", paramSizeVal, sizeof(int)); Damaris error = "
<< damaris_error_string(dam_err_) << std::endl;
resbool = false;
has_error_ = true;
}
}
if (resbool == true) {
parameterIsSet(); // sets parameters_set_ and gets the size of the
// variables block storage (as number of elemnts)
}
if (total_size > 0) {
current_size_ = total_size;
} else {
dam_err_sstr_ << " ERROR rank =" << rank_ << " : class DamarisVar::getDataStoreBlockSize() "
<< "The total size of the variable is 0 - please check "
"input paramSizeVal array."
<< std::endl;
has_error_ = true;
}
if (hasError()) {
printError();
}
}
/**
* Method to set the Damaris position values.
*
* /param [IN] positionsVals : An pointer to a value or array of values to
* set as the offset into the array. One element per dimension (one value for
* each dim_)
*
* /implicit : Implicitly uses the variable name: \ref
* variable_name_
*/
void setDamarisPosition(const std::vector<int64_t>& positionsVals)
{
assert(positionsVals.size() == num_params_);
for (int pos_dim = 0; pos_dim < num_params_; pos_dim++) {
positions_[pos_dim] = positionsVals[pos_dim];
}
dam_err_ = damaris_set_position(variable_name_.c_str(), positionsVals.data());
if (dam_err_ != DAMARIS_OK) {
dam_err_sstr_ << " ERROR rank =" << rank_ << " : class DamarisVar : damaris_set_position(\""
<< variable_name_
<< "\", positionsVals); Damaris error = " << damaris_error_string(dam_err_) << std::endl;
has_error_ = true;
}
if (hasError()) {
printError();
}
}
/**
* Method to set the internal pointer (data_ptr_) to the Damaris shared
* memory area.
*
* /implicit : Implicitly uses the Damaris variable name
* string \ref variable_name_ /implicit : Implicitly uses the
* class data element : \ref data_ptr_
*/
void setPointersToDamarisShmem(void)
{
if (parameters_set_ == true) {
// Allocate memory in the shared memory section...
dam_err_ = damaris_alloc(variable_name_.c_str(), (void**)&data_ptr_);
if (dam_err_ != DAMARIS_OK) {
dam_err_sstr_ << " ERROR rank =" << rank_ << " : class DamarisVar : damaris_alloc(\""
<< variable_name_ << "\", (void **) &ret_ptr)"
<< ", Damaris error = " << damaris_error_string(dam_err_) << std::endl;
has_error_ = true;
}
} else {
dam_err_ = -1;
dam_err_sstr_ << " ERROR rank =" << rank_
<< " : class DamarisVar : setDamarisParameter() should be "
"called first so as to define the size of the memory "
"block required for variable : "
<< variable_name_ << std::endl;
has_error_ = true;
}
if (hasError()) {
printError();
}
}
/**
* Method to commit the memory of the data written to the Damaris variable -
* Indicates that we will not write any more data to \ref data_ptr_
*
* /implicit : Implicitly uses the variable name string \ref
* variable_name_
*/
void commitVariableDamarisShmem(void)
{
// Signal to Damaris we are done writing data for this iteration
dam_err_ = damaris_commit(variable_name_.c_str());
if (dam_err_ != DAMARIS_OK) {
dam_err_sstr_ << " ERROR rank =" << rank_ << " : class DamarisVar : damaris_commit(\""
<< variable_name_ << "\")"
<< ", Damaris error = " << damaris_error_string(dam_err_) << std::endl;
has_error_ = true;
}
}
/**
* Method to release the memory of the data written to the Damaris variable -
* Indicates that Damaris may take control of the shared memory area that was
* used for the variable \ref data_ptr_
*
* /implicit : Implicitly uses the variable name string \ref
* variable_name_
*/
void clearVariableDamarisShmem(void)
{
// Signal to Damaris it has complete charge of the memory area
dam_err_ = damaris_clear(variable_name_.c_str());
if (dam_err_ != DAMARIS_OK) {
dam_err_sstr_ << " ERROR rank =" << rank_ << " : class DamarisVar : damaris_clear(\"" << variable_name_
<< "\")"
<< ", Damaris error = " << damaris_error_string(dam_err_) << std::endl;
has_error_ = true;
}
data_ptr_ = nullptr;
}
private:
void formatTypeError(std::string& var_name, std::string type_name1, std::string type_name2)
{
dam_err_sstr_ << " ERROR rank =" << rank_ << " : DamarisVar::DamarisVar () variable_name_: \"" << var_name
<< "\" The template type of Type of DamarisVar<T> in the code: " << type_name1
<< " does not match type in XML:" << type_name2 << std::endl;
has_error_ = true;
}
}; // class DamarisVar
} // namespace DamarisOutput
} // namespace Opm
#endif

785
opm/simulators/utils/GridDataOutput.hpp Normal file
View File

@ -0,0 +1,785 @@
/*
Copyright 2023 Inria, Bretagne–Atlantique Research Center
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPM_GRID_DATA_OUTPUT_HPP
#define OPM_GRID_DATA_OUTPUT_HPP
#include <dune/grid/common/rangegenerators.hh>
#include <dune/grid/io/file/vtk/common.hh>
#include <sstream>
/** @file
@brief Allows model geometry data to be passed to external code - via a copy
direct to input pointers.
This data extractor provides the full set of vertices (corresponding to
Dune::Partition::all) and then allows a user to specify Dune sub-partitions
to get the references into the vertex array and element (aka cell) types for
the sub-partition. This allows the full set of vertices to be reused for
visualisation of the various sub-partitions, at the expense of copying all
the vertices. Typically a user is interested in the interiorBorder elements
which make use of the bulk (~80%) of the vertices. This saves having to
renumber the indexes to the vertices for the sub-partitions. The vertex data
can be retrieved as seperate x, y and z arrays, or as a single array of
array of structures, or as single structure of arrays based array.
Example:
// From the opm-simulators repository
#include <opm/simulators/utils/GridDataOutput.hpp>
// N.B. does not seem to be able to be allocated with new operator.
Opm::GridDataOutput::SimMeshDataAccessor geomData(gridView,
Dune::Partition::interior );
geomData.printGridDetails();
int nvert = geomData.getNVertices();
// example using seperate x, y and z arrays
int nvert = geomData.getNVertices();
double * x_vert = new double[nvert];
double * y_vert = new double[nvert];
double * z_vert = new double[nvert];
geomData.writeGridPoints(x_vert,y_vert,z_vert, nvert);
... do something with vertex data x_vert, y_vert and z_vert ....
delete [] x_vert;
delete [] y_vert;
delete [] z_vert;
// example using AOS
double * xyz_vert_aos = new double[nvert*3];
geomData.writeGridPoints_AOS(xyz_vert_aos, nvert);
... do something with vertex data xyz_vert_aos....
delete [] xyz_vert_aos;
// example using SOA with std::vector<double>
std::vector<double> xyz_vert_soa(nvert*3);
geomData.writeGridPoints_SOA(xyz_vert_soa);
... do something with vertex data xyz_vert_soa....
*/
namespace Opm::GridDataOutput
{
/**
* Allows selection of order of vertices in writeConnectivity()
*/
enum ConnectivityVertexOrder { DUNE = 0, VTK = 1 };
template <class GridView, unsigned int partitions>
class SimMeshDataAccessor
{
public:
/**
* @brief Construct a SimMeshDataAccessor working on a specific GridView and
* specialize to a Dune::PartitionSet<>.
*
* @param gridView The gridView
* @param PartitionSet<> the set of cells from which to extract geometric data
*
* The PartitionSet of the data can be specified from one of:
* Dune::Partitions::all
* Dune::Partitions::interior
* Dune::Partitions::border
* Dune::Partitions::overlap
* Dune::Partitions::front
* Dune::Partitions::ghost
* Dune::Partitions::interiorBorder
* Dune::Partitions::interiorBorderOverlap
* Dune::Partitions::interiorBorderOverlapFront
* Dune::Partitions::all
*
* N.B. To visualise 'field' data on the extracted grid mesh then the field
* variable should contain at least as many vlaues as the mesh has cells
* (ncells_) or vertices (nvertices_) depending on if data is cell centred or
* vertex centred, respectively.
*
* As we are templated on the Dune::PartitionSet<partitions>, values for
* ncorners_, nvertices_ and ncells_ cannot change
*
* This class does not work with grids containing polyhedral cells (well, it
* has not been tested with this kind of grid data). The user should call
* polyhedralCellPresent() to test if polyhedral cells are present and decide
* what they want to do before copying data using the data accessor methods.
*/
explicit SimMeshDataAccessor(const GridView& gridView, Dune::PartitionSet<partitions> dunePartition)
: gridView_(gridView)
, dunePartition_(dunePartition)
{
dimw_ = GridView::dimension; // this is an enum
partition_value_ = dunePartition.value;
countEntities();
}
/**
Checks for cells that have polyhedral type within the current partition of
cells
Returns true if a polyhedral sell is found. If this is the case then this
partition is not going to be available for visualisation as this class does
not yet handle polyhedral cells.
*/
bool polyhedralCellPresent()
{
for (const auto& cit : elements(gridView_, dunePartition_)) {
auto corner_geom = cit.geometry();
if (Dune::VTK::geometryType(corner_geom.type()) == Dune::VTK::polyhedron) {
return true;
}
}
return false;
}
/**
Count the vertices, cells and corners.
Count all the vertices ( the Dune::Partitions::all partition ) as then we
do not need to renumber the vertices as all the subsets use references to
the full set.
*/
void countEntities()
{
// We include all the vertices for this ranks partition
const auto& vert_partition = vertices(gridView_, Dune::Partitions::all);
nvertices_ = std::distance(vert_partition.begin(), vert_partition.end());
const auto& cell_partition = elements(gridView_, dunePartition_);
ncells_ = 0;
ncorners_ = 0;
for (const auto& cit : cell_partition) {
auto corner_geom = cit.geometry();
ncorners_ += corner_geom.corners();
++ncells_;
}
}
/**
Write the positions of vertices - directly to the pointers given in
parameters
@param x_inout to be filled with x coordinate verticies
@param y_inout to be filled with y coordinate verticies
@param y_inout to be filled with z coordinate verticies
@param max_size the maximum number of elements of type T that can be
written to the input pointer memory regions.
Returns the number of vertices written
*/
template <typename T>
long writeGridPoints(T* x_inout, T* y_inout, T* z_inout, long max_size = 0)
{
if (max_size < nvertices_) {
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints( T& x_inout, T& "
"y_inout, T& z_inout ) "
+ " Input objects max_size (" + std::to_string(max_size)
+ ") is not sufficient to fit the nvertices_ values (" + std::to_string(nvertices_) + ")");
}
long i = 0;
if (dimw_ == 3) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0); // vertices only have one corner
x_inout[i] = static_cast<T>(xyz_local[0]);
y_inout[i] = static_cast<T>(xyz_local[1]);
z_inout[i] = static_cast<T>(xyz_local[2]);
i++;
}
} else if (dimw_ == 2) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0); // vertices only have one corner
x_inout[i] = static_cast<T>(xyz_local[0]);
y_inout[i] = static_cast<T>(xyz_local[1]);
z_inout[i] = static_cast<T>(0.0);
i++;
}
}
assert(i == nvertices_); // As we are templated on the
// Dune::PartitionSet<partitions>, this cannot change
return i;
}
/**
Write the positions of vertices - directly to the pointers given in
parameters
@param x_inout to be filled with x coordinate verticies
@param y_inout to be filled with y coordinate verticies
@param y_inout to be filled with z coordinate verticies
All parameters must have a size() and data() method (e.g. a std::vector<T>)
and the current size() must be big enough
Returns the number of vertices written
*/
template <typename VectType>
long writeGridPoints(VectType& x_inout, VectType& y_inout, VectType& z_inout)
{
size_t check_size_x = x_inout.size();
size_t check_size_y = y_inout.size();
size_t check_size_z = z_inout.size();
using VT = decltype(x_inout.data()[0]);
if ((check_size_x < nvertices_) || (check_size_y < nvertices_) || (check_size_z < nvertices_)) {
// assert(check_size >= nvertices_);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints( VectType& x_inout, VectType& "
"y_inout, VectType& z_inout ) At least one of the inputs"
+ " object x size " + std::to_string(check_size_x) + " object y size "
+ std::to_string(check_size_y) + " object z size " + std::to_string(check_size_z)
+ " is not sufficient to fit the nvertices_ values( " + std::to_string(nvertices_) + " )");
}
long i = 0;
if (dimw_ == 3) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0); // vertices only have one corner
x_inout.data()[i] = static_cast<VT>(xyz_local[0]);
y_inout.data()[i] = static_cast<VT>(xyz_local[1]);
z_inout.data()[i] = static_cast<VT>(xyz_local[2]);
i++;
}
} else if (dimw_ == 2) {
double td = 0.0;
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0); // vertices only have one corner
x_inout.data()[i] = static_cast<VT>(xyz_local[0]);
y_inout.data()[i] = static_cast<VT>(xyz_local[1]);
z_inout.data()[i] = static_cast<VT>(td);
i++;
}
}
assert(i == nvertices_); // As we are templated on the
// Dune::PartitionSet<partitions>, this cannot change
return i;
}
/**
Write the positions of vertices - directly to the pointers given in
parameters as Array of Structures x,y,z,x,y,z,x,y,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
@param max_size is the maximum number x,y,z structures with elements of type T
that can be written to the input pointer memory regions.
Returns the number of vertices written
*/
template <typename T>
long writeGridPoints_AOS(T* xyz_inout, long max_size = 0)
{
if (max_size < nvertices_ * 3) {
assert(max_size >= nvertices_ * 3);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints_AOS( T* xyz_inout ) " + " Input objects max_size ("
+ std::to_string(max_size) + ") is not sufficient to fit the nvertices_ * 3 values ("
+ std::to_string(nvertices_ * 3) + ")");
}
long i = 0;
if (dimw_ == 3) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0);
xyz_inout[i++] = static_cast<T>(xyz_local[0]);
xyz_inout[i++] = static_cast<T>(xyz_local[1]);
xyz_inout[i++] = static_cast<T>(xyz_local[2]);
}
} else if (dimw_ == 2) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0);
xyz_inout[i++] = static_cast<T>(xyz_local[0]);
xyz_inout[i++] = static_cast<T>(xyz_local[1]);
xyz_inout[i++] = static_cast<T>(0.0);
}
}
return ((i) / 3);
}
/**
Write the positions of vertices - directly to the pointers given in
parameters as Array of Structures x,y,z,x,y,z,x,y,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns the number of vertices written
*/
template <typename VectType>
long writeGridPoints_AOS(VectType& xyz_inout)
{
size_t check_size = xyz_inout.size();
using VT = decltype(xyz_inout.data()[0]);
if (check_size < nvertices_ * 3) {
assert(check_size >= nvertices_ * 3);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints_AOS( VectType& xyz_inout ) "
+ " Input objects check_size (" + std::to_string(check_size)
+ ") is not sufficient to fit the nvertices_ * 3 values (" + std::to_string(nvertices_ * 3)
+ ")");
}
long i = 0;
if (dimw_ == 3) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0);
xyz_inout.data()[i++] = static_cast<VT>(xyz_local[0]);
xyz_inout[i++] = static_cast<VT>(xyz_local[1]);
xyz_inout[i++] = static_cast<VT>(xyz_local[2]);
}
} else if (dimw_ == 2) {
double td = 0.0;
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0);
xyz_inout[i++] = static_cast<VT>(xyz_local[0]);
xyz_inout[i++] = static_cast<VT>(xyz_local[1]);
xyz_inout[i++] = static_cast<VT>(td);
}
}
return ((i) / 3);
}
/**
Write the positions of vertices - directly to the pointers given in
parameters as Structure of Arrays: x,x,x,...,y,y,y,...,z,z,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
@param max_size number of verticies (x,...y,...z,... structures) with elements of type T
that can be written to the input pointer memory regions.
Returns the number of vertices written
*/
template <typename T>
long writeGridPoints_SOA(T* xyz_inout, long max_size = 0)
{
if (max_size < nvertices_ * 3) {
// assert(max_size >= nvertices_ * 3);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints_SOA( T& xyz_inout ) " + " Input objects max_size ("
+ std::to_string(max_size) + ") is not sufficient to fit the nvertices_ * 3 values ("
+ std::to_string(nvertices_ * 3) + ")");
}
long i = 0;
// Get offsets into structure
T* xyz_inout_y = xyz_inout + nvertices_;
T* xyz_inout_z = xyz_inout + (2 * nvertices_);
if (dimw_ == 3) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0);
xyz_inout[i] = static_cast<T>(xyz_local[0]);
xyz_inout_y[i] = static_cast<T>(xyz_local[1]);
xyz_inout_z[i] = static_cast<T>(xyz_local[2]);
i++;
}
} else if (dimw_ == 2) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0);
xyz_inout[i] = static_cast<T>(xyz_local[0]);
xyz_inout_y[i] = static_cast<T>(xyz_local[1]);
xyz_inout_z[i] = static_cast<T>(0.0);
i++;
}
}
return (i);
}
/**
Write the positions of vertices - directly to the pointers given in
parameters as Structure of Arrays: x,x,x,...,y,y,y,...,z,z,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns the number of vertices written
*/
template <typename VectType>
long writeGridPoints_SOA(VectType& xyz_inout)
{
size_t check_size = xyz_inout.size();
if (check_size < nvertices_ * 3) {
// assert(check_size >= nvertices_ * 3);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints_SOA( VectType& xyz_inout ) "
+ " Input objects check_size (" + std::to_string(check_size)
+ ") is not sufficient to fit the nvertices_ * 3 values (" + std::to_string(nvertices_ * 3)
+ ")");
}
using VT = decltype(xyz_inout.data()[0]);
long i = 0;
// Get offsets into structure
VT* xyz_inout_y = xyz_inout.data() + nvertices_;
VT* xyz_inout_z = xyz_inout.data() + (2 * nvertices_);
if (dimw_ == 3) {
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0);
xyz_inout.data()[i] = static_cast<VT>(xyz_local[0]);
xyz_inout_y[i] = static_cast<VT>(xyz_local[1]);
xyz_inout_z[i] = static_cast<VT>(xyz_local[2]);
i++;
}
} else if (dimw_ == 2) {
double td = 0.0;
for (const auto& vit : vertices(gridView_, Dune::Partitions::all)) {
auto xyz_local = vit.geometry().corner(0);
xyz_inout.data()[i] = static_cast<VT>(xyz_local[0]);
xyz_inout_y[i] = static_cast<VT>(xyz_local[1]);
xyz_inout_z[i] = static_cast<VT>(td);
i++;
}
}
return (i);
}
/**
* Write the connectivity array - directly to the pointer given in parameter 1
Reorders the indices as selected either in DUNE order or VTK order.
@param connectivity_inout is the array to be filled with connectivity indexes
(i.e. the index into the vertex array)
@param whichOrder, is the order that verticies are traversed to create a cell (VTK or DUNE)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of corner values written.
Returns the number of corner indices written.
*/
template <typename Integer>
long writeConnectivity(Integer* connectivity_inout, ConnectivityVertexOrder whichOrder, long max_size = 0)
{
if (max_size < ncorners_) {
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeConnectivity( T* connectivity_inout,... ) "
+ " Input max_size value (" + std::to_string(max_size)
+ ") is not sufficient to fit the ncorners_ values (" + std::to_string(ncorners_) + ")");
}
long i = 0;
if (whichOrder == DUNE) {
// DUNE order
for (const auto& cit : elements(gridView_, dunePartition_)) {
auto cell_corners = cit.geometry().corners();
for (auto vx = 0; vx < cell_corners; ++vx) {
const int vxIdx = gridView_.indexSet().subIndex(cit, vx, 3);
connectivity_inout[i + vx] = vxIdx;
}
i += cell_corners;
}
} else {
// VTK order
for (const auto& cit : elements(gridView_, dunePartition_)) {
auto cell_corners = cit.geometry().corners();
for (auto vx = 0; vx < cell_corners; ++vx) {
const int vxIdx = gridView_.indexSet().subIndex(cit, vx, 3);
int vtkOrder;
vtkOrder = Dune::VTK::renumber(cit.type(), vx);
connectivity_inout[i + vtkOrder] = vxIdx;
}
i += cell_corners;
}
}
return (i);
}
/**
* Write the connectivity array - directly to a VectType object given in parameter 1
Reorders the indices as selected either in DUNE order or VTK order.
@param connectivity_inout is the array to be filled with connectivity indexes
(i.e. the index into the vertex array)
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
@param whichOrder, is the order that verticies are traversed to create a cell (VTK or DUNE)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of corner values written.
Returns the number of corner indices written.
*/
template <typename VectType>
long writeConnectivity(VectType& connectivity_inout, ConnectivityVertexOrder whichOrder)
{
size_t check_size = connectivity_inout.size();
if (check_size < ncorners_) {
// assert(check_size >= ncorners_);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeConnectivity( VectType& "
"connectivity_inout ) "
+ " Input objects size (" + std::to_string(check_size)
+ ") is not sufficient to fit the ncorners_ values (" + std::to_string(ncorners_) + ")");
}
long i = 0;
if (whichOrder == DUNE) {
// DUNE order
for (const auto& cit : elements(gridView_, dunePartition_)) {
auto cell_corners = cit.geometry().corners();
for (auto vx = 0; vx < cell_corners; ++vx) {
const int vxIdx = gridView_.indexSet().subIndex(cit, vx, 3);
connectivity_inout.data()[i + vx] = vxIdx;
}
i += cell_corners;
}
} else {
// VTK order
for (const auto& cit : elements(gridView_, dunePartition_)) {
auto cell_corners = cit.geometry().corners();
for (auto vx = 0; vx < cell_corners; ++vx) {
const int vxIdx = gridView_.indexSet().subIndex(cit, vx, 3);
int vtkOrder;
vtkOrder = Dune::VTK::renumber(cit.type(), vx);
connectivity_inout.data()[i + vtkOrder] = vxIdx;
}
i += cell_corners;
}
}
return (i);
}
/**
* Write the offsets values - directly to the pointer given in parameter 1
@param offsets_inout is the array to be filled with offsets into the connectivity array
(i.e. the index into the connectivity array to determine the vertices used for
the particular cell)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of cell offset values written.
Returns number of offset values written + 1
*/
template <typename Integer>
long writeOffsetsCells(Integer* offsets_inout, long max_size = 0)
{
if (max_size < ncells_) {
// assert(max_size >= ncells_);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeOffsetsCells( T* offsets_inout ) " + " Input objects max_size ("
+ std::to_string(max_size) + ") is not sufficient to fit the ncells_ values ("
+ std::to_string(ncells_) + ")");
}
long i = 1;
offsets_inout[0] = 0;
for (const auto& cit : elements(gridView_, dunePartition_)) {
auto cell_corners = cit.geometry().corners();
offsets_inout[i] = offsets_inout[i - 1] + cell_corners;
i++;
}
return (i); // This should be 1 greater than ncells_
}
/**
* Write the offsets values - directly to a VectType object given in parameter 1
@param offsets_inout is the array to be filled with offsets into the connectivity array
(i.e. the index into the connectivity array to determine the vertices used for
the particular cell).
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns number of offset values written + 1
*/
template <typename VectType>
long writeOffsetsCells(VectType& offsets_inout)
{
size_t check_size = offsets_inout.size();
if (check_size < ncells_) {
// assert(check_size >= ncells_);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeOffsetsCells( VectType& "
"offsets_inout ) "
+ " Input objects size (" + std::to_string(offsets_inout.size())
+ ") is not sufficient to fit the ncells_ values (" + std::to_string(ncells_) + ")");
}
// using VT = decltype(offsets_inout.data()[0]);
long i = 1;
offsets_inout.data()[0] = 0;
for (const auto& cit : elements(gridView_, dunePartition_)) {
auto cell_corners = cit.geometry().corners();
offsets_inout.data()[i] = offsets_inout.data()[i - 1] + cell_corners;
i++;
}
return (i); // This should be 1 greater than ncells_
}
/**
* Write the cell types values - directly to the pointer given in parameter 1
@param types_inout is the array to be filled with the cell types (VTK defined values)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of cell offset values written.
Returns number of cells type values written
*/
template <typename Integer>
long writeCellTypes(Integer* types_inout, long max_size = 0)
{
if (max_size < ncells_) {
// assert(max_size >= ncells_);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeCellTypes( T* types_inout ) " + " Input objects max_size ("
+ std::to_string(max_size) + ") is not sufficient to fit the ncells_ values ("
+ std::to_string(ncells_) + ")");
}
int i = 0;
for (const auto& cit : elements(gridView_, dunePartition_)) {
Integer vtktype = static_cast<Integer>(Dune::VTK::geometryType(cit.type()));
types_inout[i++] = vtktype;
}
return (i);
}
/**
* Write the cell types values - directly to the VectType object given in parameter 1
@param types_inout is the array to be filled with the cell types (VTK defined values)
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns number of cells type values written
*/
template <typename VectType>
long writeCellTypes(VectType& types_inout)
{
size_t check_size = types_inout.size();
if (check_size < ncells_) {
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeCellTypes( VectType& types_inout ) " + " Input objects check_size ("
+ std::to_string(check_size) + ") is not sufficient to fit the ncells_ values ("
+ std::to_string(ncells_) + ")");
}
int i = 0;
for (const auto& cit : elements(gridView_, dunePartition_)) {
int vtktype = static_cast<int>(Dune::VTK::geometryType(cit.type()));
types_inout.data()[i++] = vtktype;
}
return (i);
}
std::string getPartitionTypeString()
{
if (this->dunePartition_ == Dune::Partitions::all)
return (std::string("Dune::Partitions::all"));
if (this->dunePartition_ == Dune::Partitions::interior)
return (std::string("Dune::Partitions::interior"));
if (this->dunePartition_ == Dune::Partitions::interiorBorder)
return (std::string("Dune::Partitions::interiorBorder"));
if (this->dunePartition_ == Dune::Partitions::interiorBorderOverlap)
return (std::string("Dune::Partitions::interiorBorderOverlap"));
if (this->dunePartition_ == Dune::Partitions::front)
return (std::string("Dune::Partitions::front"));
if (this->dunePartition_ == Dune::Partitions::interiorBorderOverlapFront)
return (std::string("Dune::Partitions::InteriorBorderOverlapFront"));
if (this->dunePartition_ == Dune::Partitions::border)
return (std::string("Dune::Partitions::border"));
if (this->dunePartition_ == Dune::Partitions::ghost)
return (std::string("Dune::Partitions::ghost"));
return (std::string("Unknown Dune::PartitionSet<>"));
}
Dune::PartitionSet<partitions> getPartition(void)
{
return (this->dunePartition_);
}
void printGridDetails(std::ostream& outstr)
{
outstr << "Dune Partition = " << partition_value_ << ", " << getPartitionTypeString() << std::endl;
outstr << "ncells_: " << getNCells() << std::endl;
outstr << "nvertices_: " << getNVertices() << std::endl;
outstr << "ncorners_: " << getNCorners() << std::endl;
}
int getNCells()
{
return (ncells_);
}
int getNVertices()
{
return (nvertices_);
}
int getNCorners()
{
return (ncorners_);
}
std::string getError()
{
return error_strm_.str();
}
void clearError()
{
error_strm_.str("");
}
bool hasError()
{
if (error_strm_.str().length() > 0)
return true;
else
return false;
}
protected:
GridView gridView_; // the grid
Dune::PartitionSet<partitions> dunePartition_;
unsigned int partition_value_;
/**
Current partition grid information
*/
int ncells_;
/**
Current partition grid information
*/
int nvertices_;
/**
Current partition grid information
*/
int ncorners_;
int dimw_; // dimensions of the input grid
private:
std::stringstream error_strm_;
};
} // namespace Opm::GridDataOutput
#endif

76
opm/simulators/utils/initDamarisXmlFile.cpp
View File

@ -22,8 +22,6 @@
namespace Opm::DamarisOutput
{
/*
Below is the XML file for Damaris that is supported by Damaris.
@ -37,37 +35,36 @@ namespace Opm::DamarisOutput
std::string initDamarisXmlFile()
{
std::string init_damaris = R"V0G0N(<?xml version="1.0"?>
<simulation name="opm-flow" language="c" xmlns="http://damaris.gforge.inria.fr/damaris/model">
<simulation name="_SIM_NAME_" language="c" xmlns="http://damaris.gforge.inria.fr/damaris/model">
<architecture>
<domains count="1"/>
<dedicated cores="_DC_REGEX_" nodes="_DN_REGEX_"/>
<buffer name="buffer" size="_SHMEM_BUFFER_BYTES_REGEX_" />
<buffer name="_SHMEM_NAME_" size="_SHMEM_BUFFER_BYTES_REGEX_" />
<placement />
<queue name="queue" size="300" />
</architecture>
<data>
<data>
<parameter name="n_elements_total" type="int" value="1" />
<parameter name="n_elements_local" type="int" value="1" />
<parameter name="n" type="int" value="1" />
<layout name="zonal_layout_usmesh_integer" type="int" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="GLOBAL_CELL_INDEX" layout="zonal_layout_usmesh_integer" type="scalar" visualizable="false" time-varying="false" store="_MYSTORE_OR_EMPTY_REGEX_" centering="zonal" />
<layout name="zonal_layout_usmesh" type="double" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="PRESSURE" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="unstructured_mesh" unit="Bar" centering="zonal" time-varying="true"
select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_PYTHON_XML_NAME_" />
<layout name="zonal_layout_usmesh_integer" type="int" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="GLOBAL_CELL_INDEX" layout="zonal_layout_usmesh_integer" type="scalar" visualizable="false" time-varying="false" centering="zonal" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<layout name="zonal_layout_usmesh" type="double" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="PRESSURE" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="_PRESSURE_UNIT_" centering="zonal" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
_MORE_VARIABLES_REGEX_
<variable name="MPI_RANK" layout="zonal_layout_usmesh_integer" type="scalar" visualizable="true" mesh="us_mesh" unit="rank" centering="zonal" store="_MYSTORE_MESH_OR_EMPTY_REGEX_" time-varying="false" select-file="GLOBAL_CELL_INDEX" script="_MAKE_AVAILABLE_IN_PYTHON_" comment="The MPI rank of each cell"/>
<variable name="MPI_RANK" layout="zonal_layout_usmesh_integer" type="scalar" visualizable="true" mesh="unstructured_mesh" unit="rank" centering="zonal" select-file="GLOBAL_CELL_INDEX" store="#" time-varying="false" script="#" comment="The cells MPI rank"/>
<variable name="KRNSW_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="unstructured_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="KRNSW_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="PCSWM_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="PCSWM_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="PPCW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="RS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" comment="Dissolved Gas units Gas Oil Ratio" />
<variable name="RV" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="SOMAX" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="KRNSW_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="KRNSW_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="PCSWM_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="PCSWM_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="PPCW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="RS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" comment="Dissolved Gas units Gas Oil Ratio" />
<variable name="RV" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SOMAX" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
@ -81,8 +78,6 @@ std::string initDamarisXmlFile()
<variable name="WAT_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WAT_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="2FBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="4FBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="DFBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
@ -109,30 +104,27 @@ std::string initDamarisXmlFile()
<variable name="WIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<parameter name="n_coords_local" type="int" value="1" />
<layout name="n_coords_layout" type="double" dimensions="n_coords_local" comment="For the individual x, y and z coordinates of the mesh vertices, these values are referenced in the topologies/topo/subelements/connectivity_pg data" />
<group name="coordset/coords/values">
<variable name="x" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="y" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="z" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="x" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_MAKE_AVAILABLE_IN_PYTHON_" time-varying="false" store="_MYSTORE_MESH_OR_EMPTY_REGEX_" />
<variable name="y" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_MAKE_AVAILABLE_IN_PYTHON_" time-varying="false" store="_MYSTORE_MESH_OR_EMPTY_REGEX_" />
<variable name="z" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_MAKE_AVAILABLE_IN_PYTHON_" time-varying="false" store="_MYSTORE_MESH_OR_EMPTY_REGEX_" />
</group>
<parameter name="n_connectivity_ph" type="int" value="1" />
<layout name="n_connections_layout_ph" type="int" dimensions="n_connectivity_ph" comment="Layout for connectivities " />
<parameter name="n_offsets_types_ph" type="int" value="1" />
<layout name="n_offsets_layout_ph" type="int" dimensions="n_offsets_types_ph+1" comment="Layout for the offsets_ph" />
<layout name="n_offsets_layout_ph" type="int" dimensions="n_offsets_types_ph" comment="Layout for the offsets_ph" />
<layout name="n_types_layout_ph" type="char" dimensions="n_offsets_types_ph" comment="Layout for the types_ph " />
<group name="topologies/topo/elements">
<variable name="connectivity" layout="n_connections_layout_ph" type="scalar" visualizable="false" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="offsets" layout="n_offsets_layout_ph" type="scalar" visualizable="false" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="types" layout="n_types_layout_ph" type="scalar" visualizable="false" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="connectivity" layout="n_connections_layout_ph" type="scalar" visualizable="false" script="_MAKE_AVAILABLE_IN_PYTHON_" time-varying="false" store="_MYSTORE_MESH_OR_EMPTY_REGEX_" />
<variable name="offsets" layout="n_offsets_layout_ph" type="scalar" visualizable="false" script="_MAKE_AVAILABLE_IN_PYTHON_" time-varying="false" store="_MYSTORE_MESH_OR_EMPTY_REGEX_" />
<variable name="types" layout="n_types_layout_ph" type="scalar" visualizable="false" script="_MAKE_AVAILABLE_IN_PYTHON_" time-varying="false" store="_MYSTORE_MESH_OR_EMPTY_REGEX_" />
</group>
<mesh name="unstructured_mesh" type="unstructured" topology="3" time-varying="false"
<mesh name="us_mesh" type="unstructured" topology="3" time-varying="false"
comment="This Mesh definition is for connection with Paraview.
This definition references the variables that define an unstructured mesh specified above." >
<coord name="coordset/coords/values/x" unit="m" />
@ -147,7 +139,7 @@ std::string initDamarisXmlFile()
<polyhedral_cell_faces_offsets name="#" />
<polyhedral_n_faces_per_cell name="#" />
</mesh>
</data>
<storage>
@ -159,17 +151,17 @@ std::string initDamarisXmlFile()
</storage>
<scripts>
<pyscript name="PythonScript" file="_PYTHON_OR_EMPTY_REGEX_" language="python" frequency="1" scheduler-file="" nthreads="0" keep-workers="no" />
<_DISABLEPYTHONSTART_pyscript name="PythonScript" file="_PYTHON_SCRIPT_" language="python" frequency="1" scheduler-file="_DASK_SCHEDULER_FILE_" nthreads="0" keep-workers="no" /_DISABLEPYTHONFIN_>
</scripts>
<!-- paraview update-frequency="1" write-vtk="0" write-vtk-binary="false">
<script>_PARAVIEWPY_OR_EMPTY_REGEX_</script>
</paraview -->
<_DISABLEPARAVIEWSTART_paraview update-frequency="1" write-vtk="0" write-vtk-binary="false" >
<script>_PARAVIEW_PYTHON_SCRIPT_</script>
</paraview _DISABLEPARAVIEWFIN_>
<actions>
</actions>
<log FileName="_PATH_REGEX_/damaris_log/opm-flow" RotationSize="5" LogFormat="[%TimeStamp%]: %Message%" Flush="true" LogLevel="info" />
<log FileName="_PATH_REGEX_/damaris_log/_SIM_NAME_" RotationSize="5" LogFormat="[%TimeStamp%]: %Message%" Flush="_LOG_FLUSH_" LogLevel="_LOG_LEVEL_" />
</simulation>)V0G0N";