OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

finalise the PR 4889 modifications - added command line option to specifiy the shared memory file name in the <buffer> XML element --damaris-shared-memory-name which will be a randomised string opm-damaris-<random-string> for better multi-simulation behaviour when running more than one simulation on a single node.

Browse Source
This commit is contained in:
Josh Bowden 2023-12-13 17:50:05 +01:00
parent 757a96941a
commit e16cd9a3b3
7 changed files with 347 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
ebos/damariswriter.hh
View File

@ -99,6 +99,10 @@ struct DamarisDedicatedNodes {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisSharedMemoryName {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisSharedMemorySizeBytes {
using type = UndefinedProperty;
};
@ -152,39 +156,41 @@ public:
{
EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisOutputHdfCollective,
"Write output via Damaris using parallel HDF5 to get single file and dataset per timestep instead of one per Damaris \n \
core with multiple datasets.");
core with multiple datasets.");
EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisSaveToHdf,
"Set to false to prevent output to HDF5. Uses collective output by default or set --enable-damaris-collective=false to\n \
use file per core (file per Damaris server).");
use file per core (file per Damaris server).");
EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisSaveMeshToHdf,
"Saves the mesh data to the HDF5 file (1st iteration only). Will set --damaris-output-hdf-collective to false \n \
so will use file per core (file per Damaris server) output (global sizes and offset values \n \
of mesh variables are not being provided as yet).");
so will use file per core (file per Damaris server) output (global sizes and offset values \n \
of mesh variables are not being provided as yet).");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisPythonScript,
"Set to the path and filename of a Python script to run on Damaris server resources with access to OPM flow data.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisPythonParaviewScript,
"Set to the path and filename of a Paraview Python script to run on Paraview Catalyst (1 or 2) on Damaris server \n \
resources with access to OPM flow data.");
resources with access to OPM flow data.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSimName,
"The name of the simulation to be used by Damaris. If empty (the default) then Damaris uses \"opm-sim-<magic_number>\". \n \
This name should preferably be unique as it is used for the Damaris shmem name and by the Python Dask \n \
library to locate sections of variables.");
"The name of the simulation to be used by Damaris. If empty (the default) then Damaris uses \"opm-sim-<random-number>\". \n \
This name is used for the Damaris HDF5 file name prefix. Make unique if writing to the same output directory.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisLogLevel,
"The log level for the Damaris logging system (boost log based). \n \
Levels are: [trace, debug, info, warning, error, fatal]. Currently debug and info are useful. ");
Levels are: [trace, debug, info, warning, error, fatal]. Currently debug and info are useful. ");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisDaskFile,
"The name of a Dask json configuration file (if using Dask for processing).");
EWOMS_REGISTER_PARAM(TypeTag, int, DamarisDedicatedCores,
"Set the number of dedicated cores (MPI processes) that should be used for Damaris processing (per node). \n \
Must divide evenly into the number of simulation ranks (client ranks).");
Must divide evenly into the number of simulation ranks (client ranks).");
EWOMS_REGISTER_PARAM(TypeTag, int, DamarisDedicatedNodes,
"Set the number of dedicated nodes (full nodes) that should be used for Damaris processing (per simulation). \n \
Must divide evenly into the number of simulation nodes.");
Must divide evenly into the number of simulation nodes.");
EWOMS_REGISTER_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes,
"Set the size of the shared memory buffer used for IPC between the simulation and the Damaris resources. \n \
Needs to hold all the variables published, possibly over multiple simulation iterations.");
Needs to hold all the variables published, possibly over multiple simulation iterations.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSharedMemoryName,
"The name of the shared memory area to be used by Damaris for the current. If empty (the default) then Damaris uses \"opm-damaris-<random-string>\". \n \
This name should be unique if multiple simulations are running on the same node/server as it is used for the Damaris shmem name and by the Python Dask \n \
library to locate sections of variables.");
}
// The Simulator object should preferably have been const - the
@ -270,7 +276,7 @@ public:
int64_t temp_int64_t[1];
temp_int64_t[0] = static_cast<int64_t>(this->elements_rank_offsets_[rank_]);
dam_err_ = damaris_set_position("PRESSURE", temp_int64_t);
if (dam_err_ != DAMARIS_OK && rank_ == 0) {
if (dam_err_ != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})"
"damaris_set_position(PRESSURE, ...), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) ));
@ -331,24 +337,20 @@ private:
"( rank:{}) damaris_write(GLOBAL_CELL_INDEX, ...), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) ));
}
// This is an example of writing to the Damaris shared memory directly (i.e. not using
// damaris_write() to copy data there)
// We will add the MPI rank value directly into shared memory using the DamarisVar
// wrapper of the C based Damaris API.
// The shared memory is given back to Damaris on object deletion - i.e. when the
// unique_ptr goes out of scope.
std::unique_ptr<DamarisVarInt> mpi_rank_var( new DamarisVarInt(1,
{std::string("n_elements_local")},
std::string("MPI_RANK"), rank_) );
// N.B. we have not set any offset values, so HDF5 collective and Dask arrays cannot be used.
mpi_rank_var->setDamarisParameterAndShmem( {this->numElements_ } ) ;
// The shared memory is given back to Damaris when the DamarisVarInt goes out of scope.
DamarisVarInt mpi_rank_var_test(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_);
mpi_rank_var_test.setDamarisParameterAndShmem( {this->numElements_ } ) ;
// Fill the created memory area
for (int i = 0 ; i < this->numElements_; i++ )
{
mpi_rank_var->data()[i] = rank_ ; // write the rank vaue to the shared memory area.
mpi_rank_var_test.data()[i] = rank_ ; // write the rank vaue to the shared memory area.
}
}
void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid, std::vector<unsigned long long>& elements_rank_offsets)
@ -417,10 +419,9 @@ private:
"damaris_set_position(\"GLOBAL_CELL_INDEX\", temp_int64_t);");
}
std::unique_ptr<DamarisVarInt> mpi_rank_var( new DamarisVarInt(1,
{std::string("n_elements_local")},
std::string("MPI_RANK"), rank_) ) ;
mpi_rank_var->setDamarisPosition({*temp_int64_t}) ;
// Set the size of the MPI variable
DamarisVarInt mpi_rank_var(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_) ;
mpi_rank_var.setDamarisPosition({*temp_int64_t}) ;
}
@ -448,27 +449,23 @@ private:
// <variable name="z" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="PythonConduitTest" time-varying="false" />
// </group>
std::unique_ptr<DamarisVarDbl> var_x(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/x"), rank_)) ;
DamarisVarDbl var_x(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/x"), rank_) ;
// N.B. We have not set any position/offset values (using DamarisVar::SetDamarisPosition).
// They are not needed for mesh data as each process has a local geometric model.
// However, HDF5 collective and Dask arrays cannot be used for this data.
var_x->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
var_x.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
std::unique_ptr<DamarisVarDbl> var_y(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/y"), rank_)) ;
var_y->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
DamarisVarDbl var_y(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/y"), rank_) ;
var_y.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
std::unique_ptr<DamarisVarDbl> var_z(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/z"), rank_)) ;
var_z->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
DamarisVarDbl var_z(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/z"), rank_) ;
var_z.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
// Now we can return the memory that Damaris has allocated in shmem and use it to write the X,y,z coordinates
double itime, ftime, exec_time;
itime = omp_get_wtime();
if ( geomData.writeGridPoints(*var_x,*var_y,*var_z) < 0)
if ( geomData.writeGridPoints(var_x, var_y, var_z) < 0)
DUNE_THROW(Dune::IOError, geomData.getError() );
if ( geomData.writeGridPoints(var_x->data(),var_y->data(),var_z->data(), geomData.getNVertices()) < 0)
DUNE_THROW(Dune::IOError, geomData.getError() );
ftime = omp_get_wtime();
exec_time = ftime - itime;
// OpmLog::info("\n\nTime taken geomData.writeGridPoints(): is " + std::to_string(exec_time) ) ;
@ -488,29 +485,28 @@ private:
// <variable name="types" layout="n_types_layout_ph" type="scalar" visualizable="false" unit="" script="PythonConduitTest" time-varying="false" />
// </group>
std::unique_ptr<DamarisVarInt> var_connectivity(new DamarisVarInt(1, {std::string("n_connectivity_ph")}, std::string("topologies/topo/elements/connectivity"), rank_)) ;
var_connectivity->setDamarisParameterAndShmem({ geomData.getNCorners()}) ;
std::unique_ptr<DamarisVarInt> var_offsets(new DamarisVarInt(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/offsets"), rank_)) ;
var_offsets->setDamarisParameterAndShmem({ geomData.getNCells()}) ;
std::unique_ptr<DamarisVarChar> var_types(new DamarisVarChar(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/types"), rank_)) ;
var_types->setDamarisParameterAndShmem({ geomData.getNCells()}) ;
DamarisVarInt var_connectivity(1, {std::string("n_connectivity_ph")}, std::string("topologies/topo/elements/connectivity"), rank_) ;
var_connectivity.setDamarisParameterAndShmem({ geomData.getNCorners()}) ;
DamarisVarInt var_offsets(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/offsets"), rank_) ;
var_offsets.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
DamarisVarChar var_types(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/types"), rank_) ;
var_types.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
// Copy the mesh data from the Durne grid
long i = 0 ;
Opm::GridDataOutput::ConnectivityVertexOrder vtkorder = Opm::GridDataOutput::VTK ;
i = geomData.writeConnectivity(*var_connectivity, vtkorder) ;
i = geomData.writeConnectivity(var_connectivity, vtkorder) ;
if ( i != geomData.getNCorners())
DUNE_THROW(Dune::IOError, geomData.getError());
i = geomData.writeOffsetsCells(*var_offsets);
i = geomData.writeOffsetsCells(var_offsets);
if ( i != geomData.getNCells()+1)
DUNE_THROW(Dune::IOError,geomData.getError());
i = geomData.writeCellTypes(*var_types) ;
i = geomData.writeCellTypes(var_types) ;
if ( i != geomData.getNCells())
DUNE_THROW(Dune::IOError,geomData.getError());
}
catch (std::exception& e)
{

4
ebos/eclproblem_properties.hh
View File

@ -426,6 +426,10 @@ struct DamarisDedicatedNodes<TypeTag, TTag::EclBaseProblem> {
static constexpr int value = 0;
};
template<class TypeTag>
struct DamarisSharedMemoryName<TypeTag, TTag::EclBaseProblem> {
static constexpr auto value = "" ; // default name is empty, will make unique if needed in DamarisKeywords()
};
template<class TypeTag>
struct DamarisSharedMemorySizeBytes<TypeTag, TTag::EclBaseProblem> {
static constexpr long value = 536870912; // 512 MB
};

82
opm/simulators/utils/DamarisKeywords.cpp
View File

@ -77,11 +77,27 @@ bool FileExists(const std::string& filename_in,
return (file_exists);
}
void DamarisSettings::SetRandString(void)
{
// rand_value_str_ = damaris::Environment::GetMagicNumber(comm); // requires Damaris >= v1.9.2
// We will create a random value.
// Seed with a real random value, if available
std::random_device r;
// Choose a random number between 0 and MAX_INT
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, std::numeric_limits<int>::max());
int rand_int = uniform_dist(e1);
rand_value_str_ = std::to_string(rand_int) ;
}
std::map<std::string, std::string>
DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& comm,
const std::string& OutputDir)
{
SetRandString() ; // sets rand_value_str_ used for naming things that might need a unique name
std::string saveToHDF5_str("MyStore");
if (! saveToDamarisHDF5_ ){
saveToHDF5_str = "#";
@ -126,8 +142,9 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
}
}
#else
OpmLog::info(fmt::format("INFO: Opm::DamarisOutput::DamarisKeywords() : Paraview is not enabled in the Damaris library. "
"The commandline --damaris-python-paraview-script={} will be set to empty string", paraviewPythonFilename_));
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : Paraview is not enabled in the Damaris library. "
"The commandline --damaris-python-paraview-script={} will be set to empty string",
paraviewPythonFilename_));
paraviewPythonFilename_.clear();
#endif
@ -154,29 +171,40 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
} else {
damarisOutputCollective_str = "FilePerCore";
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <option key=\"FileMode\"> {} </option> ",
damarisOutputCollective_str));
std::string simName_str;
// Check if simulation name was given on command line
// The simulation name is used as a prefix to name HDF5 files
if (damarisSimName_.empty()) {
// Having a different simulation name is important if multiple simulations
// are running on the same node, as it is used to name the simulations shmem area
// and when one sim finishes it removes its shmem file.
// simName_str = damaris::Environment::GetMagicNumber(comm);
if (simName_str.empty()) {
// We will add a random value as GetMagicNumber(comm) requires Damaris v1.9.2
// Seed with a real random value, if available
std::random_device r;
// Choose a random number between 0 and MAX_INT
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, std::numeric_limits<int>::max());
int rand_int = uniform_dist(e1);
simName_str = "opm-flow-" + std::to_string(rand_int);
} else {
simName_str = "opm-flow-" + simName_str;
}
simName_str = "opm-flow-" + rand_value_str_;
} else {
simName_str = damarisSimName_;
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <simulation name={} ",
simName_str));
// A different shared memory buffer name is important if multiple simulations
// are running on the same node, as one simulation will remove the buffer when it exits,
// which will remove the buffer for other simulations.
std::string shmemName_str;
if ( shmemName_.empty()) {
shmemName_str = "opm-damaris-" + rand_value_str_;
} else {
shmemName_str = shmemName_ ;
}
std::string shmemSizeBytes_str;
if (shmemSizeBytes_ != 0) {
shmemSizeBytes_str = std::to_string(shmemSizeBytes_);
} else {
shmemSizeBytes_str = "536870912"; // 512 MB
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <buffer name={} size={} ",
shmemName_str, shmemSizeBytes_str));
if ((nDamarisCores_ > 0) && (nDamarisNodes_ > 0))
{
nDamarisNodes_ = 0; // Default is to use Damaris Cores
@ -194,20 +222,20 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
} else {
nDamarisNodes_str = "0";
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <dedicated cores={} nodes={} ",
nDamarisCores_str, nDamarisNodes_str));
std::string shmemSizeBytes_str;
if (shmemSizeBytes_ != 0) {
shmemSizeBytes_str = std::to_string(shmemSizeBytes_);
} else {
shmemSizeBytes_str = "536870912"; // 512 MB
}
std::string logLevel_str(damarisLogLevel_);
std::string logFlush_str("false");
if ((logLevel_str == "debug") || (logLevel_str == "trace") ) {
logFlush_str = "true";
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <log FileName={}/damaris_log/{} Flush={} LogLevel={} ",
OutputDir, simName_str, logFlush_str, logLevel_str));
std::map<std::string, std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_", shmemSizeBytes_str},
{"_DC_REGEX_", nDamarisCores_str},
@ -222,6 +250,7 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
{"_PRESSURE_UNIT_","Pa"},
{"_MAKE_AVAILABLE_IN_PYTHON_",publishToPython_str}, /* must match <pyscript name="PythonScript" */
{"_SIM_NAME_",simName_str},
{"_SHMEM_NAME_",shmemName_str},
{"_LOG_LEVEL_",logLevel_str},
{"_LOG_FLUSH_",logFlush_str},
{"_DISABLEPYTHONSTART_",disablePythonXMLstart},
@ -230,6 +259,7 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
{"_DISABLEPARAVIEWFIN_",disableParaviewXMLfin},
{"_DASK_SCHEDULER_FILE_",damarisDaskFile_},
};
return damaris_keywords;
}

8
opm/simulators/utils/DamarisKeywords.hpp
View File

@ -57,16 +57,21 @@ struct DamarisSettings {
std::string pythonFilename_;
std::string paraviewPythonFilename_;
std::string damarisSimName_; // empty defaults to opm-sim-<magic_number>
std::string damarisSimName_; // empty and set to "opm-flow-<random-number>" if none provided on command line. Used as prefix to HDF5 filenames
std::string shmemName_; // empty and needs to be unique if multiple simulations are running on the same server/node. Used to name the Damaris shared memory region.
std::string damarisLogLevel_ = "info";
std::string damarisDaskFile_ = "";
int nDamarisCores_ = 1; // this is the number of (Damaris server) cores per node
int nDamarisNodes_ = 0;
long shmemSizeBytes_ = 536870912; // 512 MB
std::string rand_value_str_ ; // to be added to sheared memory name to make unique
std::map<std::string, std::string>
getKeywords(const Parallel::Communication& comm,
const std::string& OutputDir);
void SetRandString(void); // sets the value of rand_value_str_
};
/**
@ -93,6 +98,7 @@ getDamarisKeywords(const Parallel::Communication& comm, const std::string& Outpu
settings.nDamarisCores_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedCores);
settings.nDamarisNodes_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedNodes);
settings.shmemSizeBytes_ = EWOMS_GET_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes);
settings.shmemName_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisSharedMemoryName);
settings.damarisLogLevel_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisLogLevel);
settings.damarisDaskFile_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisDaskFile);
return settings.getKeywords(comm, OutputDir);

206
opm/simulators/utils/DamarisVar.hpp
View File

@ -161,38 +161,14 @@ public:
}; // class DamarisVarBase
/**
* class to store a Damaris variable representation for the XML file (can be
* used with /ref class DamarisKeywords).
*
* It is thought that the details stored in the object can be used to pass into
* an XML generation function e.g. DamarisKeywords
*
* Should be instantiated with something like the following:
* // The following needs to be defined in the Damaris XML file:
* // <parameter name="n_elements_mpi_local" type="int" value="1" />
* // <layout name="mpi_layout" type="int"
* dimensions="n_elements_mpi_local" comment="MPI elements layout" />
* // <variable name="MPI_RANK" layout="mpi_layout" type="scalar"
* visualizable="true" mesh="unstructured_mesh" unit="rank" centering="zonal"
* // store="#"
* time-varying="false" script="_PYTHON_XML_NAME_" comment="The cells MPI
* rank"/> damaris::model::DamarisVar<int> dam_var = new
* damaris::model::DamarisVar<int>(1, {std::string("n_connectivity_ph")},
* std::string("topologies/topo/elements/connectivity"), rank_);
* dam_var->SetDamarisParameterAndShmem( { geomData.getNCorners() } );
*
* int * shmem_mpi_ptr = dam_var->data_ptr();
* // Fill the created memory area
* for (int i = 0; i <; i++ )
* {
* shmem_mpi_ptr[i] = rank_;
* }
* delete dam_var; // this tells Damaris that the shared memory that it
* supplied is at its disposal. It will print error messages too.
*
*/
* class to store a Damaris variable representation for the XML file
* (can be used with /ref class DamarisKeywords).
*
* It is thought that the details stored in the object can be used to pass
* into an XML generation function e.g. DamarisKeywords
*
*/
template <typename T> class DamarisVar : public DamarisVarBase {
int dims_;
int num_params_; //!< Each paramater name string will need a value and they
//!< are set in SetDamarisParameter()
std::vector<int>
@ -226,55 +202,67 @@ template <typename T> class DamarisVar : public DamarisVarBase {
size_t
current_size_; //!< The total number of elements that may be held by this
//!< part of the variable - returned by the size() method.
//!< N.B. the actual size of the data area is dependent on
//!< how the <variable> XML is written, as paramaters can
//!< be augmented by basic maths relationships. This value
//!< may not even be initialised if ParameterIsSet() method
//!< is being used (e.g. in version 2/ of the constructor below).
public:
/**
* Constructor - sets private data values and dos not initialise the shared memory area.
*
* Two usages:
* Example XML definition:
* <parameter name="my_param_name1" type="int" value="1" />
* <parameter name="my_param_name2" type="int" value="1" />
* <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" comment="This is a 2D variable"/>
* <variable name="MYVARNAME" layout="my_layout" visualizable="true"/>
*
* 1/ The variable's layout needs to be initialised via parameters :
* // Create the DamarisVar object:
* damaris::model::DamarisVar<int>* MYVARNAME_2d = new damaris::model::DamarisVar<int>(2,
* {std::string("my_param_name1"), std::string("my_param_name2")},
* {100, 25},
* std::string("MYVARNAME"), rank_);
* // sets the paramater sizes (so, here, my_param_name1 == 25 and my_param_name2 == 100)
* MYVARNAME_2d->SetDamarisParameterAndShmem( {25, 100 } };
* // Get a pointer to the memeory and use it
* T * mymemory = MYVARNAME_2d->data();
* ... write data to mymemory ....
* delete MYVARNAME_2d;
* or,
* 2/ The variable's layout has been initialised via parameters in another variable
* (i.e. "my_param_name1" and "my_param_name2" have been previously set in the code)
* // Create the DamarisVar object:
* damaris::model::DamarisVar<int>* MYVARNAME_2d = new damaris::model::DamarisVar<int>(2,
* {std::string("my_param_name1"), std::string("my_param_name2")},
* std::string("MYVARNAME"), rank_);
* // explicitly state that the paramater values have been set somewhere else in the code previously.
* MYVARNAME_2d->ParameterIsSet();
* MYVARNAME_2d->SetPointersToDamarisShmem()
* // Get a pointer to the memeory and use it
* T * mymemory = MYVARNAME_2d->data();
* ... write data to mymemory ....
* delete MYVARNAME_2d;
*
* /param [IN] dims Used to check that the inputs to SetDamarisPosition()
* have the same number of values - one value for each dimension
* /param [IN] param_names The name the Damaris paramaters. These names (in typical use) control
* a Damaris variables size (names are defined in the Damaris XML file).
* /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file)
* /param [IN] rank The rank of the process. Used for error output.
*/
/**
* Constructor - sets private data values and dos not initialise the shared memory area.
*
* N.B. These objects need a matching <variable ...> and <paramater ...> in the Damaris XML file
*
* Two usages:
* Example XML definition:
* <parameter name="my_param_name1" type="int" value="1" />
* <parameter name="my_param_name2" type="int" value="1" />
* <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" />
* <variable name="MYVARNAME" layout="my_layout" visualizable="true"/>
*
* 1/ The variable's layout needs to be initialised via parameters :
* // Create the DamarisVar object:
* damaris::model::DamarisVar<int> MYVARNAME_2d(2,{std::string("my_param_name1"),
* std::string("my_param_name2")},
* {100, 25},
* std::string("MYVARNAME"), rank_);
* // sets the paramater sizes (so, here, my_param_name1 == 25 and my_param_name2 == 100)
* MYVARNAME_2d.SetDamarisParameterAndShmem( {25, 100 } };
* // Get a pointer to the memeory and use it
* T * mymemory = MYVARNAME_2d.data();
* ... write data to mymemory ....
* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
* or,
* 2/ The variable's layout has been initialised via parameters in another variable
* (i.e. "my_param_name1" and "my_param_name2" have been previously set in the code)
* // Create the DamarisVar object:
* damaris::model::DamarisVar<int> MYVARNAME_2d(2, {std::string("my_param_name1"),
std::string("my_param_name2")},
* std::string("MYVARNAME"), rank_);
*
* // explicitly state that the paramater values have been set somewhere else in the code previously.
* MYVARNAME_2d.ParameterIsSet();
*
* N.B. This will not set the internal current_size_ value so the size() value will
* not be correct <- This is important to remember
*
* MYVARNAME_2d.SetPointersToDamarisShmem()
* // Get a pointer to the memeory and use it
* T * mymemory = MYVARNAME_2d.data();
* ... write data to mymemory ....
* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
*
* /param [IN] dims Used to check that the inputs to SetDamarisPosition()
* have the same number of values - one value for each dimension
* /param [IN] param_names The name the Damaris paramaters. These names (in typical use) control
* a Damaris variables size (names are defined in the Damaris XML file).
* /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file)
* /param [IN] rank The rank of the process. Used for error output.
*/
DamarisVar(int dims, std::vector<std::string> param_names,
std::string variable_name, int rank)
: dims_(dims), param_names_(param_names), variable_name_(variable_name),
: param_names_(param_names), variable_name_(variable_name),
rank_(rank) {
dam_err_ = DAMARIS_OK;
@ -300,38 +288,40 @@ public:
has_error_ = false;
}
/**
* Constructor - Sets private data values and also initialises the Damaris shared memory area for writing (and reading)
* by specifying the values for the variables parameters . i.e. makes the data_ptr()
*
* Example use:
* Example XML definition:
* <parameter name="my_param_name1" type="int" value="1" />
* <parameter name="my_param_name2" type="int" value="1" />
* <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" comment="This is a 2D variable" />
* <variable name="MYVARNAME" layout="my_layout" visualizable="true"/>
* // The paramaters are intialized in the constructor code
* damaris::model::DamarisVar<int>* MYVARNAME_2d = new damaris::model::DamarisVar<int>(2,
* {std::string("my_param_name1"), std::string("my_param_name2")},
* {100, 25},
* std::string("MYVARNAME"), rank_);
* T * mymemory = MYVARNAME_2d->data();
* ... write data to mymemory ....
* delete MYVARNAME_2d;
*
* /param [IN] dims Used to check that the inputs to SetDamarisPosition() have
* the same number of values - one value for each dimension
* /param [IN] param_names The name the Damaris paramaters. These names (in typical use)
* control a Damaris variables size (names are defined in the Damaris XML file).
* /param [IN] param_values The values of the paramaters - this defines how much memory we will
* have access to in the shared memory area (on the current and ongoing iterations,
* until later modified to new values)
* /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file)
* /param [IN] rank The rank of the process. Used for error output.
*/
/**
* Constructor - Sets private data values and also initialises the Damaris shared memory area for writing (and reading)
* by specifying the values for the variables parameters .
* i.e. makes the data() pointer available and sets the size of the memory block it points to.
*
* N.B. These objects need a matching <variable ...> and <paramater ...> in the Damaris XML file
*
* Example use:
* Example XML definition:
* <parameter name="my_param_name1" type="int" value="1" />
* <parameter name="my_param_name2" type="int" value="1" />
* <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" comment="This is a 2D variable" />
* <variable name="MYVARNAME" layout="my_layout" visualizable="true"/>
* // The paramaters are intialized in the constructor code
* damaris::model::DamarisVar<int> MYVARNAME_2d(2,{std::string("my_param_name1"), std::string("my_param_name2")},
* {100, 25},
* std::string("MYVARNAME"), rank_);
* T * mymemory = MYVARNAME_2d.data();
* ... write data to mymemory ....
* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
*
* /param [IN] dims Used to check that the inputs to SetDamarisPosition() have
* the same number of values - one value for each dimension
* /param [IN] param_names The name the Damaris paramaters. These names (in typical use)
* control a Damaris variables size (names are defined in the Damaris XML file).
* /param [IN] param_values The values of the paramaters - this defines how much memory we will
* have access to in the shared memory area (on the current and ongoing iterations,
* until later modified to new values)
* /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file)
* /param [IN] rank The rank of the process. Used for error output.
*/
DamarisVar(int dims, std::vector<std::string> param_names,
std::vector<int> param_values, std::string variable_name, int rank)
: dims_(dims), param_names_(param_names), variable_name_(variable_name),
: param_names_(param_names), variable_name_(variable_name),
rank_(rank) {
DamarisVar(dims, param_names, variable_name, rank);
setDamarisParameterAndShmem(
@ -589,9 +579,9 @@ public:
* variable_name_
*/
void setDamarisPosition(const std::vector<int64_t> &positionsVals) {
assert(positionsVals.size() == dims_);
assert(positionsVals.size() == num_params_);
for (int pos_dim = 0; pos_dim < dims_; pos_dim++) {
for (int pos_dim = 0; pos_dim < num_params_; pos_dim++) {
positions_[pos_dim] = positionsVals[pos_dim];
}
dam_err_ =

189
opm/simulators/utils/GridDataOutput.hpp
View File

@ -48,7 +48,7 @@
// N.B. does not seem to be able to be allocated with new operator.
Opm::GridDataOutput::SimMeshDataAccessor geomData(gridView,
Dune::Partition::interior );
Dune::Partition::interior );
geomData.printGridDetails();
@ -58,7 +58,7 @@
double * x_vert = new double[nvert];
double * y_vert = new double[nvert];
double * z_vert = new double[nvert];
geomData.writeGridPoints(x_vert,y_vert,z_vert);
geomData.writeGridPoints(x_vert,y_vert,z_vert, nvert);
... do something with vertex data x_vert, y_vert and z_vert ....
@ -68,11 +68,18 @@
// example using AOS
double * xyz_vert_aos = new double[nvert*3];
geomData.writeGridPoints_AOS(xyz_vert_aos);
geomData.writeGridPoints_AOS(xyz_vert_aos, nvert);
... do something with vertex data xyz_vert_aos....
delete [] xyz_vert_aos;
// example using SOA with std::vector<double>
std::vector<double> xyz_vert_soa(nvert*3);
geomData.writeGridPoints_SOA(xyz_vert_soa);
... do something with vertex data xyz_vert_soa....
*/
@ -148,8 +155,8 @@ public:
Count the vertices, cells and corners.
Count all the vertices ( the Dune::Partitions::all partition ) as then we
do not need to renumber the vertices as all the subsets use references to
the full set.
do not need to renumber the vertices as all the subsets use references to
the full set.
*/
void countEntities() {
// We include all the vertices for this ranks partition
@ -166,10 +173,22 @@ public:
}
}
/**
Write the positions of vertices - directly to the pointers given in
parameters
@param x_inout to be filled with x coordinate verticies
@param y_inout to be filled with y coordinate verticies
@param y_inout to be filled with z coordinate verticies
@param max_size the maximum number of elements of type T that can be
written to the input pointer memory regions.
Returns the number of vertices written
*/
template <typename T>
long writeGridPoints(T *x_inout, T *y_inout, T *z_inout, long max_size = 0) {
if (max_size < nvertices_) {
// assert(max_size >= nvertices_);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints( T& x_inout, T& "
"y_inout, T& z_inout ) " +
@ -207,21 +226,32 @@ public:
/**
Write the positions of vertices - directly to the pointers given in
parameters
@param x_inout to be filled with x coordinate verticies
@param y_inout to be filled with y coordinate verticies
@param y_inout to be filled with z coordinate verticies
All parameters must have a size() and data() method (e.g. a std::vector<T>)
and the current size() must be big enough
Returns the number of vertices written
*/
template <typename T>
long writeGridPoints(T &x_inout, T &y_inout, T &z_inout) {
size_t check_size = x_inout.size();
template <typename VectType>
long writeGridPoints(VectType &x_inout, VectType &y_inout, VectType &z_inout) {
size_t check_size_x = x_inout.size();
size_t check_size_y = y_inout.size();
size_t check_size_z = z_inout.size();
using VT = decltype(x_inout.data()[0]);
if (check_size < nvertices_) {
if ((check_size_x < nvertices_) || (check_size_y < nvertices_) || (check_size_z < nvertices_)){
// assert(check_size >= nvertices_);
OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints( T& x_inout, T& "
"y_inout, T& z_inout ) " +
" Input objects size " + std::to_string(check_size) +
"Opm::GridDataOutput::writeGridPoints( VectType& x_inout, VectType& "
"y_inout, VectType& z_inout ) At least one of the inputs" +
" object x size " + std::to_string(check_size_x) +
" object y size " + std::to_string(check_size_y) +
" object z size " + std::to_string(check_size_z) +
" is not sufficient to fit the nvertices_ values( " +
std::to_string(nvertices_) + " )");
}
@ -254,10 +284,15 @@ public:
}
/**
Write positions of vertices as array of structures : x,y,z,x,y,z,x,y,z,...
Returns the number of vertices written
*/
Write the positions of vertices - directly to the pointers given in
parameters as Array of Structures x,y,z,x,y,z,x,y,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
@param max_size is the maximum number x,y,z structures with elements of type T
that can be written to the input pointer memory regions.
Returns the number of vertices written
*/
template <typename T>
long writeGridPoints_AOS(T *xyz_inout, long max_size = 0) {
if (max_size < nvertices_ * 3) {
@ -288,10 +323,15 @@ public:
return ((i) / 3);
}
/**
Write positions of vertices as array of structures : x,y,z,x,y,z,x,y,z,...
Returns the number of vertices written
Write the positions of vertices - directly to the pointers given in
parameters as Array of Structures x,y,z,x,y,z,x,y,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns the number of vertices written
*/
template <typename VectType> long writeGridPoints_AOS(VectType &xyz_inout) {
size_t check_size = xyz_inout.size();
@ -328,12 +368,16 @@ public:
return ((i) / 3);
}
/**
Write positions of vertices as structure of arrays :
x,x,x,...,y,y,y,...,z,z,z,...
Returns the number of vertices written
*/
/**
Write the positions of vertices - directly to the pointers given in
parameters as Structure of Arrays: x,x,x,...,y,y,y,...,z,z,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
@param max_size number of verticies (x,...y,...z,... structures) with elements of type T
that can be written to the input pointer memory regions.
Returns the number of vertices written
*/
template <typename T>
long writeGridPoints_SOA(T *xyz_inout, long max_size = 0) {
if (max_size < nvertices_ * 3) {
@ -371,11 +415,14 @@ public:
}
/**
Write positions of vertices as structure of arrays :
x,x,x,...,y,y,y,...,z,z,z,...
Returns the number of vertices written
*/
Write the positions of vertices - directly to the pointers given in
parameters as Structure of Arrays: x,x,x,...,y,y,y,...,z,z,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns the number of vertices written
*/
template <typename VectType> long writeGridPoints_SOA(VectType &xyz_inout) {
size_t check_size = xyz_inout.size();
@ -420,7 +467,12 @@ public:
/**
* Write the connectivity array - directly to the pointer given in parameter 1
Reorders the indices as selected either in DUNE order or VTK order.
@param connectivity_inout is the array to be filled with connectivity indexes
(i.e. the index into the vertex array)
@param whichOrder, is the order that verticies are traversed to create a cell (VTK or DUNE)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of corner values written.
Returns the number of corner indices written.
*/
template <typename Integer>
@ -465,11 +517,17 @@ public:
}
/**
* Write the connectivity array - directly to the pointer given in parameter 1
Reorders the indices as selected either in DUNE order or VTK order.
Returns the number of corner indices written.
*/
* Write the connectivity array - directly to a VectType object given in parameter 1
Reorders the indices as selected either in DUNE order or VTK order.
@param connectivity_inout is the array to be filled with connectivity indexes
(i.e. the index into the vertex array)
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
@param whichOrder, is the order that verticies are traversed to create a cell (VTK or DUNE)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of corner values written.
Returns the number of corner indices written.
*/
template <typename VectType>
long writeConnectivity(VectType &connectivity_inout,
ConnectivityVertexOrder whichOrder) {
@ -514,10 +572,18 @@ public:
return (i);
}
/**
/**
* Write the offsets values - directly to the pointer given in parameter 1
Returns the (number of offset values written + 1)
*/
@param offsets_inout is the array to be filled with offsets into the connectivity array
(i.e. the index into the connectivity array to determine the vertices used for
the particular cell)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of cell offset values written.
Returns number of offset values written + 1
*/
template <typename Integer>
long writeOffsetsCells(Integer *offsets_inout, long max_size = 0) {
if (max_size < ncells_) {
@ -539,10 +605,16 @@ public:
return (i); // This should be 1 greater than ncells_
}
/**
* Write the offsets values - directly to the pointer given in parameter 1
Returns the (number of offset values written + 1)
*/
/**
* Write the offsets values - directly to a VectType object given in parameter 1
@param offsets_inout is the array to be filled with offsets into the connectivity array
(i.e. the index into the connectivity array to determine the vertices used for
the particular cell).
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns number of offset values written + 1
*/
template <typename VectType> long writeOffsetsCells(VectType &offsets_inout) {
size_t check_size = offsets_inout.size();
if (check_size < ncells_) {
@ -567,9 +639,16 @@ public:
return (i); // This should be 1 greater than ncells_
}
/**
* Write the Cell types array - directly to the pointer given in parameter 1
*/
/**
* Write the cell types values - directly to the pointer given in parameter 1
@param types_inout is the array to be filled with the cell types (VTK defined values)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of cell offset values written.
Returns number of cells type values written
*/
template <typename Integer>
long writeCellTypes(Integer *types_inout, long max_size = 0) {
if (max_size < ncells_) {
@ -589,14 +668,18 @@ public:
return (i);
}
/**
* Write the Cell types array - directly to the pointer given in parameter 1
*/
/**
* Write the cell types values - directly to the VectType object given in parameter 1
@param types_inout is the array to be filled with the cell types (VTK defined values)
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns number of cells type values written
*/
template <typename VectType> long writeCellTypes(VectType &types_inout) {
size_t check_size = types_inout.size();
if (check_size < ncells_) {
// assert(check_size >= ncells_);
OPM_THROW(
std::runtime_error,
"Opm::GridDataOutput::writeCellTypes( VectType& types_inout ) " +

2
opm/simulators/utils/initDamarisXmlFile.cpp
View File

@ -39,7 +39,7 @@ std::string initDamarisXmlFile()
<architecture>
<domains count="1"/>
<dedicated cores="_DC_REGEX_" nodes="_DN_REGEX_"/>
<buffer name="_SIM_NAME_" size="_SHMEM_BUFFER_BYTES_REGEX_" />
<buffer name="_SHMEM_NAME_" size="_SHMEM_BUFFER_BYTES_REGEX_" />
<placement />
<queue name="queue" size="300" />
</architecture>