OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

finalise the PR 4889 modifications - added command line option to specifiy the shared memory file name in the <buffer> XML element --damaris-shared-memory-name which will be a randomised string opm-damaris-<random-string> for better multi-simulation behaviour when running more than one simulation on a single node.

Browse Source
This commit is contained in:
Josh Bowden 2023-12-13 17:50:05 +01:00
parent 757a96941a
commit e16cd9a3b3
7 changed files with 347 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
ebos/damariswriter.hh
View File

@ -99,6 +99,10 @@ struct DamarisDedicatedNodes {
using type = UndefinedProperty; using type = UndefinedProperty;
}; };
template<class TypeTag, class MyTypeTag> template<class TypeTag, class MyTypeTag>
struct DamarisSharedMemoryName {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisSharedMemorySizeBytes { struct DamarisSharedMemorySizeBytes {
using type = UndefinedProperty; using type = UndefinedProperty;
}; };
@ -166,9 +170,8 @@ public:
"Set to the path and filename of a Paraview Python script to run on Paraview Catalyst (1 or 2) on Damaris server \n \ "Set to the path and filename of a Paraview Python script to run on Paraview Catalyst (1 or 2) on Damaris server \n \
resources with access to OPM flow data."); resources with access to OPM flow data.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSimName, EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSimName,
"The name of the simulation to be used by Damaris. If empty (the default) then Damaris uses \"opm-sim-<magic_number>\". \n \ "The name of the simulation to be used by Damaris. If empty (the default) then Damaris uses \"opm-sim-<random-number>\". \n \
This name should preferably be unique as it is used for the Damaris shmem name and by the Python Dask \n \ This name is used for the Damaris HDF5 file name prefix. Make unique if writing to the same output directory.");
library to locate sections of variables.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisLogLevel, EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisLogLevel,
"The log level for the Damaris logging system (boost log based). \n \ "The log level for the Damaris logging system (boost log based). \n \
Levels are: [trace, debug, info, warning, error, fatal]. Currently debug and info are useful. "); Levels are: [trace, debug, info, warning, error, fatal]. Currently debug and info are useful. ");
@ -184,7 +187,10 @@ public:
EWOMS_REGISTER_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes, EWOMS_REGISTER_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes,
"Set the size of the shared memory buffer used for IPC between the simulation and the Damaris resources. \n \ "Set the size of the shared memory buffer used for IPC between the simulation and the Damaris resources. \n \
Needs to hold all the variables published, possibly over multiple simulation iterations."); Needs to hold all the variables published, possibly over multiple simulation iterations.");
EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSharedMemoryName,
"The name of the shared memory area to be used by Damaris for the current. If empty (the default) then Damaris uses \"opm-damaris-<random-string>\". \n \
This name should be unique if multiple simulations are running on the same node/server as it is used for the Damaris shmem name and by the Python Dask \n \
library to locate sections of variables.");
} }
// The Simulator object should preferably have been const - the // The Simulator object should preferably have been const - the
@ -270,7 +276,7 @@ public:
int64_t temp_int64_t[1]; int64_t temp_int64_t[1];
temp_int64_t[0] = static_cast<int64_t>(this->elements_rank_offsets_[rank_]); temp_int64_t[0] = static_cast<int64_t>(this->elements_rank_offsets_[rank_]);
dam_err_ = damaris_set_position("PRESSURE", temp_int64_t); dam_err_ = damaris_set_position("PRESSURE", temp_int64_t);
if (dam_err_ != DAMARIS_OK && rank_ == 0) { if (dam_err_ != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})" OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})"
"damaris_set_position(PRESSURE, ...), Damaris Error: {} ", "damaris_set_position(PRESSURE, ...), Damaris Error: {} ",
rank_, damaris_error_string(dam_err_) )); rank_, damaris_error_string(dam_err_) ));
@ -336,19 +342,15 @@ private:
// damaris_write() to copy data there) // damaris_write() to copy data there)
// We will add the MPI rank value directly into shared memory using the DamarisVar // We will add the MPI rank value directly into shared memory using the DamarisVar
// wrapper of the C based Damaris API. // wrapper of the C based Damaris API.
// The shared memory is given back to Damaris on object deletion - i.e. when the // The shared memory is given back to Damaris when the DamarisVarInt goes out of scope.
// unique_ptr goes out of scope. DamarisVarInt mpi_rank_var_test(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_);
std::unique_ptr<DamarisVarInt> mpi_rank_var( new DamarisVarInt(1, mpi_rank_var_test.setDamarisParameterAndShmem( {this->numElements_ } ) ;
{std::string("n_elements_local")},
std::string("MPI_RANK"), rank_) );
// N.B. we have not set any offset values, so HDF5 collective and Dask arrays cannot be used.
mpi_rank_var->setDamarisParameterAndShmem( {this->numElements_ } ) ;
// Fill the created memory area // Fill the created memory area
for (int i = 0 ; i < this->numElements_; i++ ) for (int i = 0 ; i < this->numElements_; i++ )
{ {
mpi_rank_var->data()[i] = rank_ ; // write the rank vaue to the shared memory area. mpi_rank_var_test.data()[i] = rank_ ; // write the rank vaue to the shared memory area.
} }
} }
void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid, std::vector<unsigned long long>& elements_rank_offsets) void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid, std::vector<unsigned long long>& elements_rank_offsets)
@ -417,10 +419,9 @@ private:
"damaris_set_position(\"GLOBAL_CELL_INDEX\", temp_int64_t);"); "damaris_set_position(\"GLOBAL_CELL_INDEX\", temp_int64_t);");
} }
std::unique_ptr<DamarisVarInt> mpi_rank_var( new DamarisVarInt(1, // Set the size of the MPI variable
{std::string("n_elements_local")}, DamarisVarInt mpi_rank_var(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_) ;
std::string("MPI_RANK"), rank_) ) ; mpi_rank_var.setDamarisPosition({*temp_int64_t}) ;
mpi_rank_var->setDamarisPosition({*temp_int64_t}) ;
} }
@ -448,27 +449,23 @@ private:
// <variable name="z" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="PythonConduitTest" time-varying="false" /> // <variable name="z" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="PythonConduitTest" time-varying="false" />
// </group> // </group>
std::unique_ptr<DamarisVarDbl> var_x(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/x"), rank_)) ; DamarisVarDbl var_x(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/x"), rank_) ;
// N.B. We have not set any position/offset values (using DamarisVar::SetDamarisPosition). // N.B. We have not set any position/offset values (using DamarisVar::SetDamarisPosition).
// They are not needed for mesh data as each process has a local geometric model. // They are not needed for mesh data as each process has a local geometric model.
// However, HDF5 collective and Dask arrays cannot be used for this data. // However, HDF5 collective and Dask arrays cannot be used for this data.
var_x->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; var_x.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
std::unique_ptr<DamarisVarDbl> var_y(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/y"), rank_)) ; DamarisVarDbl var_y(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/y"), rank_) ;
var_y->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; var_y.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
std::unique_ptr<DamarisVarDbl> var_z(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/z"), rank_)) ; DamarisVarDbl var_z(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/z"), rank_) ;
var_z->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; var_z.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
// Now we can return the memory that Damaris has allocated in shmem and use it to write the X,y,z coordinates // Now we can return the memory that Damaris has allocated in shmem and use it to write the X,y,z coordinates
double itime, ftime, exec_time; double itime, ftime, exec_time;
itime = omp_get_wtime(); itime = omp_get_wtime();
if ( geomData.writeGridPoints(*var_x,*var_y,*var_z) < 0) if ( geomData.writeGridPoints(var_x, var_y, var_z) < 0)
DUNE_THROW(Dune::IOError, geomData.getError() ); DUNE_THROW(Dune::IOError, geomData.getError() );
if ( geomData.writeGridPoints(var_x->data(),var_y->data(),var_z->data(), geomData.getNVertices()) < 0)
DUNE_THROW(Dune::IOError, geomData.getError() );
ftime = omp_get_wtime(); ftime = omp_get_wtime();
exec_time = ftime - itime; exec_time = ftime - itime;
// OpmLog::info("\n\nTime taken geomData.writeGridPoints(): is " + std::to_string(exec_time) ) ; // OpmLog::info("\n\nTime taken geomData.writeGridPoints(): is " + std::to_string(exec_time) ) ;
@ -488,29 +485,28 @@ private:
// <variable name="types" layout="n_types_layout_ph" type="scalar" visualizable="false" unit="" script="PythonConduitTest" time-varying="false" /> // <variable name="types" layout="n_types_layout_ph" type="scalar" visualizable="false" unit="" script="PythonConduitTest" time-varying="false" />
// </group> // </group>
std::unique_ptr<DamarisVarInt> var_connectivity(new DamarisVarInt(1, {std::string("n_connectivity_ph")}, std::string("topologies/topo/elements/connectivity"), rank_)) ; DamarisVarInt var_connectivity(1, {std::string("n_connectivity_ph")}, std::string("topologies/topo/elements/connectivity"), rank_) ;
var_connectivity->setDamarisParameterAndShmem({ geomData.getNCorners()}) ; var_connectivity.setDamarisParameterAndShmem({ geomData.getNCorners()}) ;
std::unique_ptr<DamarisVarInt> var_offsets(new DamarisVarInt(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/offsets"), rank_)) ; DamarisVarInt var_offsets(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/offsets"), rank_) ;
var_offsets->setDamarisParameterAndShmem({ geomData.getNCells()}) ; var_offsets.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
std::unique_ptr<DamarisVarChar> var_types(new DamarisVarChar(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/types"), rank_)) ; DamarisVarChar var_types(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/types"), rank_) ;
var_types->setDamarisParameterAndShmem({ geomData.getNCells()}) ; var_types.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
// Copy the mesh data from the Durne grid // Copy the mesh data from the Durne grid
long i = 0 ; long i = 0 ;
Opm::GridDataOutput::ConnectivityVertexOrder vtkorder = Opm::GridDataOutput::VTK ; Opm::GridDataOutput::ConnectivityVertexOrder vtkorder = Opm::GridDataOutput::VTK ;
i = geomData.writeConnectivity(*var_connectivity, vtkorder) ; i = geomData.writeConnectivity(var_connectivity, vtkorder) ;
if ( i != geomData.getNCorners()) if ( i != geomData.getNCorners())
DUNE_THROW(Dune::IOError, geomData.getError()); DUNE_THROW(Dune::IOError, geomData.getError());
i = geomData.writeOffsetsCells(*var_offsets); i = geomData.writeOffsetsCells(var_offsets);
if ( i != geomData.getNCells()+1) if ( i != geomData.getNCells()+1)
DUNE_THROW(Dune::IOError,geomData.getError()); DUNE_THROW(Dune::IOError,geomData.getError());
i = geomData.writeCellTypes(*var_types) ; i = geomData.writeCellTypes(var_types) ;
if ( i != geomData.getNCells()) if ( i != geomData.getNCells())
DUNE_THROW(Dune::IOError,geomData.getError()); DUNE_THROW(Dune::IOError,geomData.getError());
} }
catch (std::exception& e) catch (std::exception& e)
{ {

4
ebos/eclproblem_properties.hh
View File

@ -426,6 +426,10 @@ struct DamarisDedicatedNodes<TypeTag, TTag::EclBaseProblem> {
static constexpr int value = 0; static constexpr int value = 0;
}; };
template<class TypeTag> template<class TypeTag>
struct DamarisSharedMemoryName<TypeTag, TTag::EclBaseProblem> {
static constexpr auto value = "" ; // default name is empty, will make unique if needed in DamarisKeywords()
};
template<class TypeTag>
struct DamarisSharedMemorySizeBytes<TypeTag, TTag::EclBaseProblem> { struct DamarisSharedMemorySizeBytes<TypeTag, TTag::EclBaseProblem> {
static constexpr long value = 536870912; // 512 MB static constexpr long value = 536870912; // 512 MB
}; };

78
opm/simulators/utils/DamarisKeywords.cpp
View File

@ -77,11 +77,27 @@ bool FileExists(const std::string& filename_in,
return (file_exists); return (file_exists);
} }
void DamarisSettings::SetRandString(void)
{
// rand_value_str_ = damaris::Environment::GetMagicNumber(comm); // requires Damaris >= v1.9.2
// We will create a random value.
// Seed with a real random value, if available
std::random_device r;
// Choose a random number between 0 and MAX_INT
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, std::numeric_limits<int>::max());
int rand_int = uniform_dist(e1);
rand_value_str_ = std::to_string(rand_int) ;
}
std::map<std::string, std::string> std::map<std::string, std::string>
DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& comm, DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& comm,
const std::string& OutputDir) const std::string& OutputDir)
{ {
SetRandString() ; // sets rand_value_str_ used for naming things that might need a unique name
std::string saveToHDF5_str("MyStore"); std::string saveToHDF5_str("MyStore");
if (! saveToDamarisHDF5_ ){ if (! saveToDamarisHDF5_ ){
saveToHDF5_str = "#"; saveToHDF5_str = "#";
@ -126,8 +142,9 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
} }
} }
#else #else
OpmLog::info(fmt::format("INFO: Opm::DamarisOutput::DamarisKeywords() : Paraview is not enabled in the Damaris library. " OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : Paraview is not enabled in the Damaris library. "
"The commandline --damaris-python-paraview-script={} will be set to empty string", paraviewPythonFilename_)); "The commandline --damaris-python-paraview-script={} will be set to empty string",
paraviewPythonFilename_));
paraviewPythonFilename_.clear(); paraviewPythonFilename_.clear();
#endif #endif
@ -154,28 +171,39 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
} else { } else {
damarisOutputCollective_str = "FilePerCore"; damarisOutputCollective_str = "FilePerCore";
} }
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <option key=\"FileMode\"> {} </option> ",
damarisOutputCollective_str));
std::string simName_str; std::string simName_str;
// Check if simulation name was given on command line
// The simulation name is used as a prefix to name HDF5 files
if (damarisSimName_.empty()) { if (damarisSimName_.empty()) {
// Having a different simulation name is important if multiple simulations simName_str = "opm-flow-" + rand_value_str_;
// are running on the same node, as it is used to name the simulations shmem area
// and when one sim finishes it removes its shmem file.
// simName_str = damaris::Environment::GetMagicNumber(comm);
if (simName_str.empty()) {
// We will add a random value as GetMagicNumber(comm) requires Damaris v1.9.2
// Seed with a real random value, if available
std::random_device r;
// Choose a random number between 0 and MAX_INT
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, std::numeric_limits<int>::max());
int rand_int = uniform_dist(e1);
simName_str = "opm-flow-" + std::to_string(rand_int);
} else {
simName_str = "opm-flow-" + simName_str;
}
} else { } else {
simName_str = damarisSimName_; simName_str = damarisSimName_;
} }
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <simulation name={} ",
simName_str));
// A different shared memory buffer name is important if multiple simulations
// are running on the same node, as one simulation will remove the buffer when it exits,
// which will remove the buffer for other simulations.
std::string shmemName_str;
if ( shmemName_.empty()) {
shmemName_str = "opm-damaris-" + rand_value_str_;
} else {
shmemName_str = shmemName_ ;
}
std::string shmemSizeBytes_str;
if (shmemSizeBytes_ != 0) {
shmemSizeBytes_str = std::to_string(shmemSizeBytes_);
} else {
shmemSizeBytes_str = "536870912"; // 512 MB
}
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <buffer name={} size={} ",
shmemName_str, shmemSizeBytes_str));
if ((nDamarisCores_ > 0) && (nDamarisNodes_ > 0)) if ((nDamarisCores_ > 0) && (nDamarisNodes_ > 0))
{ {
@ -195,18 +223,18 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
nDamarisNodes_str = "0"; nDamarisNodes_str = "0";
} }
std::string shmemSizeBytes_str; OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <dedicated cores={} nodes={} ",
if (shmemSizeBytes_ != 0) { nDamarisCores_str, nDamarisNodes_str));
shmemSizeBytes_str = std::to_string(shmemSizeBytes_);
} else {
shmemSizeBytes_str = "536870912"; // 512 MB
}
std::string logLevel_str(damarisLogLevel_); std::string logLevel_str(damarisLogLevel_);
std::string logFlush_str("false"); std::string logFlush_str("false");
if ((logLevel_str == "debug") || (logLevel_str == "trace") ) { if ((logLevel_str == "debug") || (logLevel_str == "trace") ) {
logFlush_str = "true"; logFlush_str = "true";
} }
OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : <log FileName={}/damaris_log/{} Flush={} LogLevel={} ",
OutputDir, simName_str, logFlush_str, logLevel_str));
std::map<std::string, std::string> damaris_keywords = { std::map<std::string, std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_", shmemSizeBytes_str}, {"_SHMEM_BUFFER_BYTES_REGEX_", shmemSizeBytes_str},
@ -222,6 +250,7 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
{"_PRESSURE_UNIT_","Pa"}, {"_PRESSURE_UNIT_","Pa"},
{"_MAKE_AVAILABLE_IN_PYTHON_",publishToPython_str}, /* must match <pyscript name="PythonScript" */ {"_MAKE_AVAILABLE_IN_PYTHON_",publishToPython_str}, /* must match <pyscript name="PythonScript" */
{"_SIM_NAME_",simName_str}, {"_SIM_NAME_",simName_str},
{"_SHMEM_NAME_",shmemName_str},
{"_LOG_LEVEL_",logLevel_str}, {"_LOG_LEVEL_",logLevel_str},
{"_LOG_FLUSH_",logFlush_str}, {"_LOG_FLUSH_",logFlush_str},
{"_DISABLEPYTHONSTART_",disablePythonXMLstart}, {"_DISABLEPYTHONSTART_",disablePythonXMLstart},
@ -230,6 +259,7 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com
{"_DISABLEPARAVIEWFIN_",disableParaviewXMLfin}, {"_DISABLEPARAVIEWFIN_",disableParaviewXMLfin},
{"_DASK_SCHEDULER_FILE_",damarisDaskFile_}, {"_DASK_SCHEDULER_FILE_",damarisDaskFile_},
}; };
return damaris_keywords; return damaris_keywords;
} }

8
opm/simulators/utils/DamarisKeywords.hpp
View File

@ -57,16 +57,21 @@ struct DamarisSettings {
std::string pythonFilename_; std::string pythonFilename_;
std::string paraviewPythonFilename_; std::string paraviewPythonFilename_;
std::string damarisSimName_; // empty defaults to opm-sim-<magic_number> std::string damarisSimName_; // empty and set to "opm-flow-<random-number>" if none provided on command line. Used as prefix to HDF5 filenames
std::string shmemName_; // empty and needs to be unique if multiple simulations are running on the same server/node. Used to name the Damaris shared memory region.
std::string damarisLogLevel_ = "info"; std::string damarisLogLevel_ = "info";
std::string damarisDaskFile_ = ""; std::string damarisDaskFile_ = "";
int nDamarisCores_ = 1; // this is the number of (Damaris server) cores per node int nDamarisCores_ = 1; // this is the number of (Damaris server) cores per node
int nDamarisNodes_ = 0; int nDamarisNodes_ = 0;
long shmemSizeBytes_ = 536870912; // 512 MB long shmemSizeBytes_ = 536870912; // 512 MB
std::string rand_value_str_ ; // to be added to sheared memory name to make unique
std::map<std::string, std::string> std::map<std::string, std::string>
getKeywords(const Parallel::Communication& comm, getKeywords(const Parallel::Communication& comm,
const std::string& OutputDir); const std::string& OutputDir);
void SetRandString(void); // sets the value of rand_value_str_
}; };
/** /**
@ -93,6 +98,7 @@ getDamarisKeywords(const Parallel::Communication& comm, const std::string& Outpu
settings.nDamarisCores_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedCores); settings.nDamarisCores_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedCores);
settings.nDamarisNodes_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedNodes); settings.nDamarisNodes_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedNodes);
settings.shmemSizeBytes_ = EWOMS_GET_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes); settings.shmemSizeBytes_ = EWOMS_GET_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes);
settings.shmemName_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisSharedMemoryName);
settings.damarisLogLevel_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisLogLevel); settings.damarisLogLevel_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisLogLevel);
settings.damarisDaskFile_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisDaskFile); settings.damarisDaskFile_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisDaskFile);
return settings.getKeywords(comm, OutputDir); return settings.getKeywords(comm, OutputDir);

88
opm/simulators/utils/DamarisVar.hpp
View File

@ -161,38 +161,14 @@ public:
}; // class DamarisVarBase }; // class DamarisVarBase
/** /**
* class to store a Damaris variable representation for the XML file (can be * class to store a Damaris variable representation for the XML file
* used with /ref class DamarisKeywords). * (can be used with /ref class DamarisKeywords).
* *
* It is thought that the details stored in the object can be used to pass into * It is thought that the details stored in the object can be used to pass
* an XML generation function e.g. DamarisKeywords * into an XML generation function e.g. DamarisKeywords
*
* Should be instantiated with something like the following:
* // The following needs to be defined in the Damaris XML file:
* // <parameter name="n_elements_mpi_local" type="int" value="1" />
* // <layout name="mpi_layout" type="int"
* dimensions="n_elements_mpi_local" comment="MPI elements layout" />
* // <variable name="MPI_RANK" layout="mpi_layout" type="scalar"
* visualizable="true" mesh="unstructured_mesh" unit="rank" centering="zonal"
* // store="#"
* time-varying="false" script="_PYTHON_XML_NAME_" comment="The cells MPI
* rank"/> damaris::model::DamarisVar<int> dam_var = new
* damaris::model::DamarisVar<int>(1, {std::string("n_connectivity_ph")},
* std::string("topologies/topo/elements/connectivity"), rank_);
* dam_var->SetDamarisParameterAndShmem( { geomData.getNCorners() } );
*
* int * shmem_mpi_ptr = dam_var->data_ptr();
* // Fill the created memory area
* for (int i = 0; i <; i++ )
* {
* shmem_mpi_ptr[i] = rank_;
* }
* delete dam_var; // this tells Damaris that the shared memory that it
* supplied is at its disposal. It will print error messages too.
* *
*/ */
template <typename T> class DamarisVar : public DamarisVarBase { template <typename T> class DamarisVar : public DamarisVarBase {
int dims_;
int num_params_; //!< Each paramater name string will need a value and they int num_params_; //!< Each paramater name string will need a value and they
//!< are set in SetDamarisParameter() //!< are set in SetDamarisParameter()
std::vector<int> std::vector<int>
@ -226,44 +202,56 @@ template <typename T> class DamarisVar : public DamarisVarBase {
size_t size_t
current_size_; //!< The total number of elements that may be held by this current_size_; //!< The total number of elements that may be held by this
//!< part of the variable - returned by the size() method. //!< part of the variable - returned by the size() method.
//!< N.B. the actual size of the data area is dependent on
//!< how the <variable> XML is written, as paramaters can
//!< be augmented by basic maths relationships. This value
//!< may not even be initialised if ParameterIsSet() method
//!< is being used (e.g. in version 2/ of the constructor below).
public: public:
/** /**
* Constructor - sets private data values and dos not initialise the shared memory area. * Constructor - sets private data values and dos not initialise the shared memory area.
* *
* N.B. These objects need a matching <variable ...> and <paramater ...> in the Damaris XML file
*
* Two usages: * Two usages:
* Example XML definition: * Example XML definition:
* <parameter name="my_param_name1" type="int" value="1" /> * <parameter name="my_param_name1" type="int" value="1" />
* <parameter name="my_param_name2" type="int" value="1" /> * <parameter name="my_param_name2" type="int" value="1" />
* <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" comment="This is a 2D variable"/> * <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" />
* <variable name="MYVARNAME" layout="my_layout" visualizable="true"/> * <variable name="MYVARNAME" layout="my_layout" visualizable="true"/>
* *
* 1/ The variable's layout needs to be initialised via parameters : * 1/ The variable's layout needs to be initialised via parameters :
* // Create the DamarisVar object: * // Create the DamarisVar object:
* damaris::model::DamarisVar<int>* MYVARNAME_2d = new damaris::model::DamarisVar<int>(2, * damaris::model::DamarisVar<int> MYVARNAME_2d(2,{std::string("my_param_name1"),
* {std::string("my_param_name1"), std::string("my_param_name2")}, * std::string("my_param_name2")},
* {100, 25}, * {100, 25},
* std::string("MYVARNAME"), rank_); * std::string("MYVARNAME"), rank_);
* // sets the paramater sizes (so, here, my_param_name1 == 25 and my_param_name2 == 100) * // sets the paramater sizes (so, here, my_param_name1 == 25 and my_param_name2 == 100)
* MYVARNAME_2d->SetDamarisParameterAndShmem( {25, 100 } }; * MYVARNAME_2d.SetDamarisParameterAndShmem( {25, 100 } };
* // Get a pointer to the memeory and use it * // Get a pointer to the memeory and use it
* T * mymemory = MYVARNAME_2d->data(); * T * mymemory = MYVARNAME_2d.data();
* ... write data to mymemory .... * ... write data to mymemory ....
* delete MYVARNAME_2d; * // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
* or, * or,
* 2/ The variable's layout has been initialised via parameters in another variable * 2/ The variable's layout has been initialised via parameters in another variable
* (i.e. "my_param_name1" and "my_param_name2" have been previously set in the code) * (i.e. "my_param_name1" and "my_param_name2" have been previously set in the code)
* // Create the DamarisVar object: * // Create the DamarisVar object:
* damaris::model::DamarisVar<int>* MYVARNAME_2d = new damaris::model::DamarisVar<int>(2, * damaris::model::DamarisVar<int> MYVARNAME_2d(2, {std::string("my_param_name1"),
* {std::string("my_param_name1"), std::string("my_param_name2")}, std::string("my_param_name2")},
* std::string("MYVARNAME"), rank_); * std::string("MYVARNAME"), rank_);
*
* // explicitly state that the paramater values have been set somewhere else in the code previously. * // explicitly state that the paramater values have been set somewhere else in the code previously.
* MYVARNAME_2d->ParameterIsSet(); * MYVARNAME_2d.ParameterIsSet();
* MYVARNAME_2d->SetPointersToDamarisShmem() *
* N.B. This will not set the internal current_size_ value so the size() value will
* not be correct <- This is important to remember
*
* MYVARNAME_2d.SetPointersToDamarisShmem()
* // Get a pointer to the memeory and use it * // Get a pointer to the memeory and use it
* T * mymemory = MYVARNAME_2d->data(); * T * mymemory = MYVARNAME_2d.data();
* ... write data to mymemory .... * ... write data to mymemory ....
* delete MYVARNAME_2d; * // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
* *
* /param [IN] dims Used to check that the inputs to SetDamarisPosition() * /param [IN] dims Used to check that the inputs to SetDamarisPosition()
* have the same number of values - one value for each dimension * have the same number of values - one value for each dimension
@ -274,7 +262,7 @@ public:
*/ */
DamarisVar(int dims, std::vector<std::string> param_names, DamarisVar(int dims, std::vector<std::string> param_names,
std::string variable_name, int rank) std::string variable_name, int rank)
: dims_(dims), param_names_(param_names), variable_name_(variable_name), : param_names_(param_names), variable_name_(variable_name),
rank_(rank) { rank_(rank) {
dam_err_ = DAMARIS_OK; dam_err_ = DAMARIS_OK;
@ -302,7 +290,10 @@ public:
/** /**
* Constructor - Sets private data values and also initialises the Damaris shared memory area for writing (and reading) * Constructor - Sets private data values and also initialises the Damaris shared memory area for writing (and reading)
* by specifying the values for the variables parameters . i.e. makes the data_ptr() * by specifying the values for the variables parameters .
* i.e. makes the data() pointer available and sets the size of the memory block it points to.
*
* N.B. These objects need a matching <variable ...> and <paramater ...> in the Damaris XML file
* *
* Example use: * Example use:
* Example XML definition: * Example XML definition:
@ -311,13 +302,12 @@ public:
* <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" comment="This is a 2D variable" /> * <layout name="my_layout" type="int" dimensions="my_param_name1,my_param_name2" comment="This is a 2D variable" />
* <variable name="MYVARNAME" layout="my_layout" visualizable="true"/> * <variable name="MYVARNAME" layout="my_layout" visualizable="true"/>
* // The paramaters are intialized in the constructor code * // The paramaters are intialized in the constructor code
* damaris::model::DamarisVar<int>* MYVARNAME_2d = new damaris::model::DamarisVar<int>(2, * damaris::model::DamarisVar<int> MYVARNAME_2d(2,{std::string("my_param_name1"), std::string("my_param_name2")},
* {std::string("my_param_name1"), std::string("my_param_name2")},
* {100, 25}, * {100, 25},
* std::string("MYVARNAME"), rank_); * std::string("MYVARNAME"), rank_);
* T * mymemory = MYVARNAME_2d->data(); * T * mymemory = MYVARNAME_2d.data();
* ... write data to mymemory .... * ... write data to mymemory ....
* delete MYVARNAME_2d; * // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope.
* *
* /param [IN] dims Used to check that the inputs to SetDamarisPosition() have * /param [IN] dims Used to check that the inputs to SetDamarisPosition() have
* the same number of values - one value for each dimension * the same number of values - one value for each dimension
@ -331,7 +321,7 @@ public:
*/ */
DamarisVar(int dims, std::vector<std::string> param_names, DamarisVar(int dims, std::vector<std::string> param_names,
std::vector<int> param_values, std::string variable_name, int rank) std::vector<int> param_values, std::string variable_name, int rank)
: dims_(dims), param_names_(param_names), variable_name_(variable_name), : param_names_(param_names), variable_name_(variable_name),
rank_(rank) { rank_(rank) {
DamarisVar(dims, param_names, variable_name, rank); DamarisVar(dims, param_names, variable_name, rank);
setDamarisParameterAndShmem( setDamarisParameterAndShmem(
@ -589,9 +579,9 @@ public:
* variable_name_ * variable_name_
*/ */
void setDamarisPosition(const std::vector<int64_t> &positionsVals) { void setDamarisPosition(const std::vector<int64_t> &positionsVals) {
assert(positionsVals.size() == dims_); assert(positionsVals.size() == num_params_);
for (int pos_dim = 0; pos_dim < dims_; pos_dim++) { for (int pos_dim = 0; pos_dim < num_params_; pos_dim++) {
positions_[pos_dim] = positionsVals[pos_dim]; positions_[pos_dim] = positionsVals[pos_dim];
} }
dam_err_ = dam_err_ =

129
opm/simulators/utils/GridDataOutput.hpp
View File

@ -58,7 +58,7 @@
double * x_vert = new double[nvert]; double * x_vert = new double[nvert];
double * y_vert = new double[nvert]; double * y_vert = new double[nvert];
double * z_vert = new double[nvert]; double * z_vert = new double[nvert];
geomData.writeGridPoints(x_vert,y_vert,z_vert); geomData.writeGridPoints(x_vert,y_vert,z_vert, nvert);
... do something with vertex data x_vert, y_vert and z_vert .... ... do something with vertex data x_vert, y_vert and z_vert ....
@ -68,13 +68,20 @@
// example using AOS // example using AOS
double * xyz_vert_aos = new double[nvert*3]; double * xyz_vert_aos = new double[nvert*3];
geomData.writeGridPoints_AOS(xyz_vert_aos); geomData.writeGridPoints_AOS(xyz_vert_aos, nvert);
... do something with vertex data xyz_vert_aos.... ... do something with vertex data xyz_vert_aos....
delete [] xyz_vert_aos; delete [] xyz_vert_aos;
// example using SOA with std::vector<double>
std::vector<double> xyz_vert_soa(nvert*3);
geomData.writeGridPoints_SOA(xyz_vert_soa);
... do something with vertex data xyz_vert_soa....
*/ */
namespace Opm::GridDataOutput { namespace Opm::GridDataOutput {
@ -166,10 +173,22 @@ public:
} }
} }
/**
Write the positions of vertices - directly to the pointers given in
parameters
@param x_inout to be filled with x coordinate verticies
@param y_inout to be filled with y coordinate verticies
@param y_inout to be filled with z coordinate verticies
@param max_size the maximum number of elements of type T that can be
written to the input pointer memory regions.
Returns the number of vertices written
*/
template <typename T> template <typename T>
long writeGridPoints(T *x_inout, T *y_inout, T *z_inout, long max_size = 0) { long writeGridPoints(T *x_inout, T *y_inout, T *z_inout, long max_size = 0) {
if (max_size < nvertices_) { if (max_size < nvertices_) {
// assert(max_size >= nvertices_);
OPM_THROW(std::runtime_error, OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints( T& x_inout, T& " "Opm::GridDataOutput::writeGridPoints( T& x_inout, T& "
"y_inout, T& z_inout ) " + "y_inout, T& z_inout ) " +
@ -208,20 +227,31 @@ public:
Write the positions of vertices - directly to the pointers given in Write the positions of vertices - directly to the pointers given in
parameters parameters
@param x_inout to be filled with x coordinate verticies
@param y_inout to be filled with y coordinate verticies
@param y_inout to be filled with z coordinate verticies
All parameters must have a size() and data() method (e.g. a std::vector<T>)
and the current size() must be big enough
Returns the number of vertices written Returns the number of vertices written
*/ */
template <typename T> template <typename VectType>
long writeGridPoints(T &x_inout, T &y_inout, T &z_inout) { long writeGridPoints(VectType &x_inout, VectType &y_inout, VectType &z_inout) {
size_t check_size = x_inout.size(); size_t check_size_x = x_inout.size();
size_t check_size_y = y_inout.size();
size_t check_size_z = z_inout.size();
using VT = decltype(x_inout.data()[0]); using VT = decltype(x_inout.data()[0]);
if (check_size < nvertices_) { if ((check_size_x < nvertices_) || (check_size_y < nvertices_) || (check_size_z < nvertices_)){
// assert(check_size >= nvertices_); // assert(check_size >= nvertices_);
OPM_THROW(std::runtime_error, OPM_THROW(std::runtime_error,
"Opm::GridDataOutput::writeGridPoints( T& x_inout, T& " "Opm::GridDataOutput::writeGridPoints( VectType& x_inout, VectType& "
"y_inout, T& z_inout ) " + "y_inout, VectType& z_inout ) At least one of the inputs" +
" Input objects size " + std::to_string(check_size) + " object x size " + std::to_string(check_size_x) +
" object y size " + std::to_string(check_size_y) +
" object z size " + std::to_string(check_size_z) +
" is not sufficient to fit the nvertices_ values( " + " is not sufficient to fit the nvertices_ values( " +
std::to_string(nvertices_) + " )"); std::to_string(nvertices_) + " )");
} }
@ -254,7 +284,12 @@ public:
} }
/** /**
Write positions of vertices as array of structures : x,y,z,x,y,z,x,y,z,... Write the positions of vertices - directly to the pointers given in
parameters as Array of Structures x,y,z,x,y,z,x,y,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
@param max_size is the maximum number x,y,z structures with elements of type T
that can be written to the input pointer memory regions.
Returns the number of vertices written Returns the number of vertices written
*/ */
@ -288,8 +323,13 @@ public:
return ((i) / 3); return ((i) / 3);
} }
/** /**
Write positions of vertices as array of structures : x,y,z,x,y,z,x,y,z,... Write the positions of vertices - directly to the pointers given in
parameters as Array of Structures x,y,z,x,y,z,x,y,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns the number of vertices written Returns the number of vertices written
*/ */
@ -329,8 +369,12 @@ public:
} }
/** /**
Write positions of vertices as structure of arrays : Write the positions of vertices - directly to the pointers given in
x,x,x,...,y,y,y,...,z,z,z,... parameters as Structure of Arrays: x,x,x,...,y,y,y,...,z,z,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
@param max_size number of verticies (x,...y,...z,... structures) with elements of type T
that can be written to the input pointer memory regions.
Returns the number of vertices written Returns the number of vertices written
*/ */
@ -371,8 +415,11 @@ public:
} }
/** /**
Write positions of vertices as structure of arrays : Write the positions of vertices - directly to the pointers given in
x,x,x,...,y,y,y,...,z,z,z,... parameters as Structure of Arrays: x,x,x,...,y,y,y,...,z,z,z,...
@param xyz_inout is the array to be filled with x,y,z coordinate verticies.
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns the number of vertices written Returns the number of vertices written
*/ */
@ -421,6 +468,11 @@ public:
* Write the connectivity array - directly to the pointer given in parameter 1 * Write the connectivity array - directly to the pointer given in parameter 1
Reorders the indices as selected either in DUNE order or VTK order. Reorders the indices as selected either in DUNE order or VTK order.
@param connectivity_inout is the array to be filled with connectivity indexes
(i.e. the index into the vertex array)
@param whichOrder, is the order that verticies are traversed to create a cell (VTK or DUNE)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of corner values written.
Returns the number of corner indices written. Returns the number of corner indices written.
*/ */
template <typename Integer> template <typename Integer>
@ -465,9 +517,15 @@ public:
} }
/** /**
* Write the connectivity array - directly to the pointer given in parameter 1 * Write the connectivity array - directly to a VectType object given in parameter 1
Reorders the indices as selected either in DUNE order or VTK order. Reorders the indices as selected either in DUNE order or VTK order.
@param connectivity_inout is the array to be filled with connectivity indexes
(i.e. the index into the vertex array)
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
@param whichOrder, is the order that verticies are traversed to create a cell (VTK or DUNE)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of corner values written.
Returns the number of corner indices written. Returns the number of corner indices written.
*/ */
template <typename VectType> template <typename VectType>
@ -514,9 +572,17 @@ public:
return (i); return (i);
} }
/** /**
* Write the offsets values - directly to the pointer given in parameter 1 * Write the offsets values - directly to the pointer given in parameter 1
Returns the (number of offset values written + 1)
@param offsets_inout is the array to be filled with offsets into the connectivity array
(i.e. the index into the connectivity array to determine the vertices used for
the particular cell)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of cell offset values written.
Returns number of offset values written + 1
*/ */
template <typename Integer> template <typename Integer>
long writeOffsetsCells(Integer *offsets_inout, long max_size = 0) { long writeOffsetsCells(Integer *offsets_inout, long max_size = 0) {
@ -540,8 +606,14 @@ public:
} }
/** /**
* Write the offsets values - directly to the pointer given in parameter 1 * Write the offsets values - directly to a VectType object given in parameter 1
Returns the (number of offset values written + 1)
@param offsets_inout is the array to be filled with offsets into the connectivity array
(i.e. the index into the connectivity array to determine the vertices used for
the particular cell).
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns number of offset values written + 1
*/ */
template <typename VectType> long writeOffsetsCells(VectType &offsets_inout) { template <typename VectType> long writeOffsetsCells(VectType &offsets_inout) {
size_t check_size = offsets_inout.size(); size_t check_size = offsets_inout.size();
@ -568,7 +640,14 @@ public:
} }
/** /**
* Write the Cell types array - directly to the pointer given in parameter 1 * Write the cell types values - directly to the pointer given in parameter 1
@param types_inout is the array to be filled with the cell types (VTK defined values)
@param max_size is used to check that the space available in the input pointer
parameter will fit the number of cell offset values written.
Returns number of cells type values written
*/ */
template <typename Integer> template <typename Integer>
long writeCellTypes(Integer *types_inout, long max_size = 0) { long writeCellTypes(Integer *types_inout, long max_size = 0) {
@ -590,13 +669,17 @@ public:
} }
/** /**
* Write the Cell types array - directly to the pointer given in parameter 1 * Write the cell types values - directly to the VectType object given in parameter 1
@param types_inout is the array to be filled with the cell types (VTK defined values)
The object VectType must have a size() and data() method (e.g. a std::vector<T>)
Returns number of cells type values written
*/ */
template <typename VectType> long writeCellTypes(VectType &types_inout) { template <typename VectType> long writeCellTypes(VectType &types_inout) {
size_t check_size = types_inout.size(); size_t check_size = types_inout.size();
if (check_size < ncells_) { if (check_size < ncells_) {
// assert(check_size >= ncells_);
OPM_THROW( OPM_THROW(
std::runtime_error, std::runtime_error,
"Opm::GridDataOutput::writeCellTypes( VectType& types_inout ) " + "Opm::GridDataOutput::writeCellTypes( VectType& types_inout ) " +

2
opm/simulators/utils/initDamarisXmlFile.cpp
View File

@ -39,7 +39,7 @@ std::string initDamarisXmlFile()
<architecture> <architecture>
<domains count="1"/> <domains count="1"/>
<dedicated cores="_DC_REGEX_" nodes="_DN_REGEX_"/> <dedicated cores="_DC_REGEX_" nodes="_DN_REGEX_"/>
<buffer name="_SIM_NAME_" size="_SHMEM_BUFFER_BYTES_REGEX_" /> <buffer name="_SHMEM_NAME_" size="_SHMEM_BUFFER_BYTES_REGEX_" />
<placement /> <placement />
<queue name="queue" size="300" /> <queue name="queue" size="300" />
</architecture> </architecture>