OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5352 from jcbowden/damaris-limit-variables-v1

Browse Source 
Add ability to pass multiple variables to Damaris using DamarisWriter class.
This commit is contained in:
Atgeirr Flø Rasmussen
2024-06-07 15:54:59 +02:00
committed by GitHub
parent 302503e172 6fcec07bab
commit fc906842ad
7 changed files with 219 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
opm/simulators/flow/DamarisProperties.hpp
View File

@@ -87,7 +87,10 @@ template<class TypeTag, class MyTypeTag>
struct DamarisDaskFile {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisLimitVariables {
using type = UndefinedProperty;
};
} // namespace Opm::Properties
#endif // OPM_DAMARIS_PROPERTIES_HPP

20
opm/simulators/flow/DamarisWriter.cpp
View File

@@ -42,7 +42,7 @@ int setPosition(const char* field, int rank, int64_t pos)
{
int dam_err = damaris_set_position(field, &pos);
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})"
OpmLog::warning(fmt::format("damariswriter::setPosition() : ( rank:{}) "
"damaris_set_position({}, ...), Damaris Error: {} ",
rank, field, damaris_error_string(dam_err)));
}
@@ -54,7 +54,7 @@ int setParameter(const char* field, int rank, int value)
{
int dam_err = damaris_parameter_set(field, &value, sizeof(int));
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("(rank:{}) Damaris library produced an error result for "
OpmLog::warning(fmt::format("damariswriter::setParameter() (rank:{}) "
"damaris_parameter_set(\"{}\",...)", rank, field));
}
@@ -65,7 +65,7 @@ int write(const char* field, int rank, const void* data)
{
int dam_err = damaris_write(field, data);
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
OpmLog::warning(fmt::format("damariswriter::write() : ( rank:{}) "
"damaris_write({}, ...), Damaris Error: {} ",
rank, field, damaris_error_string(dam_err)));
}
@@ -77,7 +77,7 @@ int endIteration(int rank)
{
int dam_err = damaris_end_iteration();
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
OpmLog::warning(fmt::format("damariswriter::endIteration() : ( rank:{}) "
"damaris_end_iteration(), Damaris Error: {} ",
rank, damaris_error_string(dam_err)));
}
@@ -130,17 +130,9 @@ int setupWritingPars(Parallel::Communication comm,
OPM_THROW(std::runtime_error, "setupDamarisWritingPars() n_elements_global_max "
"> std::numeric_limits<int>::max() " + std::to_string(dam_err));
}
// Use damaris_set_position to set the offset in the global size of the array.
// This is used so that output functionality (e.g. HDF5Store) knows global offsets of the data of the ranks
setPosition("PRESSURE", comm.rank(), elements_rank_offsets[comm.rank()]);
dam_err = setPosition("GLOBAL_CELL_INDEX", comm.rank(), elements_rank_offsets[comm.rank()]);
// Set the size of the MPI variable
DamarisVar<int> mpi_rank_var(1, {"n_elements_local"}, "MPI_RANK", comm.rank());
mpi_rank_var.setDamarisPosition({static_cast<int64_t>(elements_rank_offsets[comm.rank()])});
return dam_err;
}
}

113
opm/simulators/flow/DamarisWriter.hpp
View File

@@ -42,6 +42,7 @@
#include <opm/simulators/flow/FlowBaseVanguard.hpp>
#include <opm/simulators/flow/OutputBlackoilModule.hpp>
#include <opm/simulators/utils/DamarisVar.hpp>
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <opm/simulators/utils/DeferredLoggingErrorHelpers.hpp>
#include <opm/simulators/utils/GridDataOutput.hpp>
#include <opm/simulators/utils/ParallelSerialization.hpp>
@@ -53,6 +54,8 @@
#include <numeric>
#include <string>
#include <vector>
#include <unordered_set>
namespace Opm {
@@ -154,6 +157,11 @@ public:
"This name should be unique if multiple simulations are running on "
"the same node/server as it is used for the Damaris shmem name and by "
"the Python Dask library to locate sections of variables.");
Parameters::registerParam<TypeTag, Properties::DamarisLimitVariables>
("A comma separated list of variable names that a user wants to pass "
"through via DamarisOutput::DamarisWriter::writeOutput)() to the "
"damaris_write() call. This can be used to limit the number of "
"variables being passed to the Damaris plugins (Paraview, Python and HDF5)");
}
// The Simulator object should preferably have been const - the
@@ -210,6 +218,8 @@ public:
this->damarisOutputModule_ = std::make_unique<OutputBlackOilModule<TypeTag>>
(simulator, this->eclIO_->finalSummaryConfig(), this->collectOnIORank_);
}
wanted_vars_set_ = Opm::DamarisOutput::getSetOfIncludedVariables<TypeTag>();
}
/*!
@@ -247,7 +257,7 @@ public:
dam_err_ = DamarisOutput::setupWritingPars(simulator_.vanguard().grid().comm(),
numElements_, elements_rank_offsets_);
// sets data for non-time-varying variables MPI_RANK and GLOBAL_CELL_INDEX
// sets positions and data for non-time-varying variables MPI_RANK and GLOBAL_CELL_INDEX
this->setGlobalIndexForDamaris() ;
// Set the geometry data for the mesh model.
@@ -259,15 +269,73 @@ public:
this->damarisUpdate_ = false;
}
int cell_data_written = 0 ;
// Call damaris_write() for all available variables
for ( const auto& damVar : localCellData )
{
// std::map<std::string, data::CellData>
const std::string& name = damVar.first ;
// If the wanted_vars_set_ set is empty then we default to passing through
// all the variables/
if (wanted_vars_set_.count(name) || wanted_vars_set_.empty())
{
const data::CellData& dataCol = damVar.second ;
OpmLog::debug(fmt::format("Name of Damaris Variable : ( rank:{}) name: {} ", rank_, name));
// Call damaris_set_position() for all available variables
// There is an assumption that all variables are the same size, with the same offset.
// see initDamarisTemplateXmlFile.cpp for the Damaris XML descriptions.
dam_err_ = DamarisOutput::setPosition(name.c_str(), rank_,
this->elements_rank_offsets_[rank_]);
// It does not seem I can test for what type of data is present (double or int)
// in the std::variant within the data::CellData, so I will use a try catch block.
try {
if (dataCol.data<double>().size() >= static_cast<std::vector<double>::size_type>(this->numElements_)) {
dam_err_ = DamarisOutput::write(name.c_str(), rank_,
dataCol.data<double>().data()) ;
} else {
OpmLog::info(fmt::format("( rank:{}) The variable \"{}\" was found to be of a different size {} (not {}).", rank_, name, dataCol.data<double>().size(), this->numElements_ ));
}
}
catch (std::bad_variant_access const& ex) {
// Not a std::vector<double>, must be a std::vector<int>
if (dataCol.data<int>().size() >= static_cast<std::vector<int>::size_type>(this->numElements_)) {
dam_err_ = DamarisOutput::write(name.c_str(), rank_,
dataCol.data<int>().data()) ;
} else {
OpmLog::info(fmt::format("( rank:{}) The variable \"{}\" was found to be of a different size {} (not {}).", rank_, name, dataCol.data<int>().size(), this->numElements_ ));
}
}
++cell_data_written ;
}
}
if (!cell_data_written) {
OpmLog::info(fmt::format("( rank:{}) No simulation data written to the Damaris server - check --damaris-limit-variables command line option (if used) has valid variable name(s) and that the Damaris XML file contains variable names that are available in your simulation.", rank_));
} else {
OpmLog::debug(fmt::format("( rank:{}) {} Damaris Variables written to the Damaris servers", rank_, cell_data_written));
}
/*
Code for when we want to pass to Damaris the single cell 'block' data variables
auto mybloc = damarisOutputModule_->getBlockData() ;
for ( auto damVar : mybloc ) {
// std::map<std::string, data::CellData>
const std::string name = std::get<0>(damVar.first) ;
const int part = std::get<1>(damVar.first) ;
double dataCol = damVar.second ;
std::cout << "Name of Damaris Block Varaiable : (" << rank_ << ") " << name << " part : " << part << " Value : " << dataCol << std::endl ;
}
dam_err_ = DamarisOutput::endIteration(rank_);
*/
if (this->damarisOutputModule_->getPRESSURE_ptr() != nullptr)
{
dam_err_ = DamarisOutput::setPosition("PRESSURE", rank_,
this->elements_rank_offsets_[rank_]);
dam_err_ = DamarisOutput::write("PRESSURE", rank_,
this->damarisOutputModule_->getPRESSURE_ptr());
dam_err_ = DamarisOutput::endIteration(rank_);
}
} // end of ! isSubstep
}
@@ -276,6 +344,7 @@ private:
int rank_ ;
int nranks_ ;
int numElements_ ; ///< size of the unique vector elements
std::unordered_set<std::string> wanted_vars_set_ ;
Simulator& simulator_;
std::unique_ptr<OutputBlackOilModule<TypeTag>> damarisOutputModule_;
@@ -289,6 +358,18 @@ private:
void setGlobalIndexForDamaris ()
{
// Use damaris_set_position to set the offset in the global size of the array.
// This is used so that output functionality (e.g. HDF5Store) knows the global offsets of
// the data of the ranks data.
dam_err_ = DamarisOutput::setPosition("GLOBAL_CELL_INDEX", rank_, elements_rank_offsets_[rank_]);
// This is an example of writing to the Damaris shared memory directly (i.e. we allocate the
// variable directly in the shared memory region and do not use damaris_write() to copy data there.
// The shared memory is given back to Damaris when the DamarisVarInt goes out of scope.
// N.B. MPI_RANK is only saved to HDF5 if --damaris-save-mesh-to-hdf=true is specified
DamarisVarInt mpi_rank_var(1, {"n_elements_local"}, "MPI_RANK", rank_);
mpi_rank_var.setDamarisPosition({static_cast<int64_t>(elements_rank_offsets_[rank_])});
// GLOBAL_CELL_INDEX is used to reorder variable data when writing to disk
// This is enabled using select-file="GLOBAL_CELL_INDEX" in the <variable> XML tag
if (this->collectOnIORank_.isParallel()) {
@@ -302,15 +383,17 @@ private:
dam_err_ = DamarisOutput::write("GLOBAL_CELL_INDEX", rank_, local_to_global_filled.data());
}
// This is an example of writing to the Damaris shared memory directly (i.e. not using
// damaris_write() to copy data there)
// We will add the MPI rank value directly into shared memory using the DamarisVar
// wrapper of the C based Damaris API.
// The shared memory is given back to Damaris when the DamarisVarInt goes out of scope.
DamarisVarInt mpi_rank_var_test(1, {"n_elements_local"}, "MPI_RANK", rank_);
mpi_rank_var_test.setDamarisParameterAndShmem( {this->numElements_ } ) ;
mpi_rank_var.setDamarisParameterAndShmem( {this->numElements_ } ) ;
// Fill the created memory area
std::fill(mpi_rank_var_test.data(), mpi_rank_var_test.data() + numElements_, rank_);
std::fill(mpi_rank_var.data(), mpi_rank_var.data() + numElements_, rank_);
// Pass the output directory string through as a Damaris variable so that
// Python code (as an example) can use the path as required.
const auto& outputDir = simulator_.vanguard().eclState().cfg().io().getOutputDir();
if (outputDir.size() > 0) {
dam_err_ = DamarisOutput::setParameter("path_string_length", rank_, outputDir.size()) ;
dam_err_ = DamarisOutput::write("OUTPUTDIR", rank_, outputDir.c_str());
}
}
void writeDamarisGridOutput()
@@ -374,7 +457,7 @@ private:
"topologies/topo/elements/types", rank_) ;
var_types.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
// Copy the mesh data from the Durne grid
// Copy the mesh data from the Dune grid
long i = 0 ;
GridDataOutput::ConnectivityVertexOrder vtkorder = GridDataOutput::VTK ;

5
opm/simulators/flow/FlowProblemProperties.hpp
View File

@@ -387,6 +387,11 @@ template<class TypeTag>
struct DamarisDaskFile<TypeTag, TTag::FlowBaseProblem> {
static constexpr auto value = "";
};
// Specify the the exact variables to be passed through to Damaris (must exist in the XML file / intiDamarisXmlFile.cpp)
template<class TypeTag>
struct DamarisLimitVariables<TypeTag, TTag::FlowBaseProblem> {
static constexpr auto value = "";
};
#endif
// If available, write the ECL output in a non-blocking manner
template<class TypeTag>