diff --git a/ebos/damariswriter.hh b/ebos/damariswriter.hh index c6d405ccd..f94656d86 100644 --- a/ebos/damariswriter.hh +++ b/ebos/damariswriter.hh @@ -99,6 +99,10 @@ struct DamarisDedicatedNodes { using type = UndefinedProperty; }; template +struct DamarisSharedMemoryName { + using type = UndefinedProperty; +}; +template struct DamarisSharedMemorySizeBytes { using type = UndefinedProperty; }; @@ -152,39 +156,41 @@ public: { EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisOutputHdfCollective, "Write output via Damaris using parallel HDF5 to get single file and dataset per timestep instead of one per Damaris \n \ - core with multiple datasets."); + core with multiple datasets."); EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisSaveToHdf, "Set to false to prevent output to HDF5. Uses collective output by default or set --enable-damaris-collective=false to\n \ - use file per core (file per Damaris server)."); + use file per core (file per Damaris server)."); EWOMS_REGISTER_PARAM(TypeTag, bool, DamarisSaveMeshToHdf, "Saves the mesh data to the HDF5 file (1st iteration only). Will set --damaris-output-hdf-collective to false \n \ - so will use file per core (file per Damaris server) output (global sizes and offset values \n \ - of mesh variables are not being provided as yet)."); + so will use file per core (file per Damaris server) output (global sizes and offset values \n \ + of mesh variables are not being provided as yet)."); EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisPythonScript, "Set to the path and filename of a Python script to run on Damaris server resources with access to OPM flow data."); EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisPythonParaviewScript, "Set to the path and filename of a Paraview Python script to run on Paraview Catalyst (1 or 2) on Damaris server \n \ - resources with access to OPM flow data."); + resources with access to OPM flow data."); EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSimName, - "The name of the simulation to be used by Damaris. If empty (the default) then Damaris uses \"opm-sim-\". \n \ - This name should preferably be unique as it is used for the Damaris shmem name and by the Python Dask \n \ - library to locate sections of variables."); + "The name of the simulation to be used by Damaris. If empty (the default) then Damaris uses \"opm-sim-\". \n \ + This name is used for the Damaris HDF5 file name prefix. Make unique if writing to the same output directory."); EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisLogLevel, "The log level for the Damaris logging system (boost log based). \n \ - Levels are: [trace, debug, info, warning, error, fatal]. Currently debug and info are useful. "); + Levels are: [trace, debug, info, warning, error, fatal]. Currently debug and info are useful. "); EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisDaskFile, "The name of a Dask json configuration file (if using Dask for processing)."); EWOMS_REGISTER_PARAM(TypeTag, int, DamarisDedicatedCores, "Set the number of dedicated cores (MPI processes) that should be used for Damaris processing (per node). \n \ - Must divide evenly into the number of simulation ranks (client ranks)."); + Must divide evenly into the number of simulation ranks (client ranks)."); EWOMS_REGISTER_PARAM(TypeTag, int, DamarisDedicatedNodes, "Set the number of dedicated nodes (full nodes) that should be used for Damaris processing (per simulation). \n \ - Must divide evenly into the number of simulation nodes."); + Must divide evenly into the number of simulation nodes."); EWOMS_REGISTER_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes, "Set the size of the shared memory buffer used for IPC between the simulation and the Damaris resources. \n \ - Needs to hold all the variables published, possibly over multiple simulation iterations."); - + Needs to hold all the variables published, possibly over multiple simulation iterations."); + EWOMS_REGISTER_PARAM(TypeTag, std::string, DamarisSharedMemoryName, + "The name of the shared memory area to be used by Damaris for the current. If empty (the default) then Damaris uses \"opm-damaris-\". \n \ + This name should be unique if multiple simulations are running on the same node/server as it is used for the Damaris shmem name and by the Python Dask \n \ + library to locate sections of variables."); } // The Simulator object should preferably have been const - the @@ -270,7 +276,7 @@ public: int64_t temp_int64_t[1]; temp_int64_t[0] = static_cast(this->elements_rank_offsets_[rank_]); dam_err_ = damaris_set_position("PRESSURE", temp_int64_t); - if (dam_err_ != DAMARIS_OK && rank_ == 0) { + if (dam_err_ != DAMARIS_OK) { OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})" "damaris_set_position(PRESSURE, ...), Damaris Error: {} ", rank_, damaris_error_string(dam_err_) )); @@ -331,24 +337,20 @@ private: "( rank:{}) damaris_write(GLOBAL_CELL_INDEX, ...), Damaris Error: {} ", rank_, damaris_error_string(dam_err_) )); } - + // This is an example of writing to the Damaris shared memory directly (i.e. not using // damaris_write() to copy data there) // We will add the MPI rank value directly into shared memory using the DamarisVar // wrapper of the C based Damaris API. - // The shared memory is given back to Damaris on object deletion - i.e. when the - // unique_ptr goes out of scope. - std::unique_ptr mpi_rank_var( new DamarisVarInt(1, - {std::string("n_elements_local")}, - std::string("MPI_RANK"), rank_) ); - - // N.B. we have not set any offset values, so HDF5 collective and Dask arrays cannot be used. - mpi_rank_var->setDamarisParameterAndShmem( {this->numElements_ } ) ; + // The shared memory is given back to Damaris when the DamarisVarInt goes out of scope. + DamarisVarInt mpi_rank_var_test(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_); + mpi_rank_var_test.setDamarisParameterAndShmem( {this->numElements_ } ) ; // Fill the created memory area for (int i = 0 ; i < this->numElements_; i++ ) { - mpi_rank_var->data()[i] = rank_ ; // write the rank vaue to the shared memory area. + mpi_rank_var_test.data()[i] = rank_ ; // write the rank vaue to the shared memory area. } + } void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid, std::vector& elements_rank_offsets) @@ -417,10 +419,9 @@ private: "damaris_set_position(\"GLOBAL_CELL_INDEX\", temp_int64_t);"); } - std::unique_ptr mpi_rank_var( new DamarisVarInt(1, - {std::string("n_elements_local")}, - std::string("MPI_RANK"), rank_) ) ; - mpi_rank_var->setDamarisPosition({*temp_int64_t}) ; + // Set the size of the MPI variable + DamarisVarInt mpi_rank_var(1, {std::string("n_elements_local")}, std::string("MPI_RANK"), rank_) ; + mpi_rank_var.setDamarisPosition({*temp_int64_t}) ; } @@ -448,27 +449,23 @@ private: // // - std::unique_ptr var_x(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/x"), rank_)) ; + DamarisVarDbl var_x(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/x"), rank_) ; // N.B. We have not set any position/offset values (using DamarisVar::SetDamarisPosition). // They are not needed for mesh data as each process has a local geometric model. // However, HDF5 collective and Dask arrays cannot be used for this data. - var_x->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; + var_x.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; - std::unique_ptr var_y(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/y"), rank_)) ; - var_y->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; + DamarisVarDbl var_y(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/y"), rank_) ; + var_y.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; - std::unique_ptr var_z(new DamarisVarDbl(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/z"), rank_)) ; - var_z->setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; + DamarisVarDbl var_z(1, {std::string("n_coords_local")}, std::string("coordset/coords/values/z"), rank_) ; + var_z.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ; // Now we can return the memory that Damaris has allocated in shmem and use it to write the X,y,z coordinates double itime, ftime, exec_time; itime = omp_get_wtime(); - if ( geomData.writeGridPoints(*var_x,*var_y,*var_z) < 0) + if ( geomData.writeGridPoints(var_x, var_y, var_z) < 0) DUNE_THROW(Dune::IOError, geomData.getError() ); - - if ( geomData.writeGridPoints(var_x->data(),var_y->data(),var_z->data(), geomData.getNVertices()) < 0) - DUNE_THROW(Dune::IOError, geomData.getError() ); - ftime = omp_get_wtime(); exec_time = ftime - itime; // OpmLog::info("\n\nTime taken geomData.writeGridPoints(): is " + std::to_string(exec_time) ) ; @@ -488,29 +485,28 @@ private: // // - std::unique_ptr var_connectivity(new DamarisVarInt(1, {std::string("n_connectivity_ph")}, std::string("topologies/topo/elements/connectivity"), rank_)) ; - var_connectivity->setDamarisParameterAndShmem({ geomData.getNCorners()}) ; - std::unique_ptr var_offsets(new DamarisVarInt(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/offsets"), rank_)) ; - var_offsets->setDamarisParameterAndShmem({ geomData.getNCells()}) ; - std::unique_ptr var_types(new DamarisVarChar(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/types"), rank_)) ; - var_types->setDamarisParameterAndShmem({ geomData.getNCells()}) ; + DamarisVarInt var_connectivity(1, {std::string("n_connectivity_ph")}, std::string("topologies/topo/elements/connectivity"), rank_) ; + var_connectivity.setDamarisParameterAndShmem({ geomData.getNCorners()}) ; + DamarisVarInt var_offsets(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/offsets"), rank_) ; + var_offsets.setDamarisParameterAndShmem({ geomData.getNCells()}) ; + DamarisVarChar var_types(1, {std::string("n_offsets_types_ph")}, std::string("topologies/topo/elements/types"), rank_) ; + var_types.setDamarisParameterAndShmem({ geomData.getNCells()}) ; // Copy the mesh data from the Durne grid long i = 0 ; Opm::GridDataOutput::ConnectivityVertexOrder vtkorder = Opm::GridDataOutput::VTK ; - i = geomData.writeConnectivity(*var_connectivity, vtkorder) ; + i = geomData.writeConnectivity(var_connectivity, vtkorder) ; if ( i != geomData.getNCorners()) DUNE_THROW(Dune::IOError, geomData.getError()); - i = geomData.writeOffsetsCells(*var_offsets); + i = geomData.writeOffsetsCells(var_offsets); if ( i != geomData.getNCells()+1) DUNE_THROW(Dune::IOError,geomData.getError()); - i = geomData.writeCellTypes(*var_types) ; + i = geomData.writeCellTypes(var_types) ; if ( i != geomData.getNCells()) DUNE_THROW(Dune::IOError,geomData.getError()); - } catch (std::exception& e) { diff --git a/ebos/eclproblem_properties.hh b/ebos/eclproblem_properties.hh index b18518363..c0b0fcb28 100644 --- a/ebos/eclproblem_properties.hh +++ b/ebos/eclproblem_properties.hh @@ -426,6 +426,10 @@ struct DamarisDedicatedNodes { static constexpr int value = 0; }; template +struct DamarisSharedMemoryName { + static constexpr auto value = "" ; // default name is empty, will make unique if needed in DamarisKeywords() +}; +template struct DamarisSharedMemorySizeBytes { static constexpr long value = 536870912; // 512 MB }; diff --git a/opm/simulators/utils/DamarisKeywords.cpp b/opm/simulators/utils/DamarisKeywords.cpp index 6b335de93..5cc990627 100644 --- a/opm/simulators/utils/DamarisKeywords.cpp +++ b/opm/simulators/utils/DamarisKeywords.cpp @@ -77,11 +77,27 @@ bool FileExists(const std::string& filename_in, return (file_exists); } +void DamarisSettings::SetRandString(void) +{ + // rand_value_str_ = damaris::Environment::GetMagicNumber(comm); // requires Damaris >= v1.9.2 + + // We will create a random value. + // Seed with a real random value, if available + std::random_device r; + // Choose a random number between 0 and MAX_INT + std::default_random_engine e1(r()); + std::uniform_int_distribution uniform_dist(0, std::numeric_limits::max()); + int rand_int = uniform_dist(e1); + + rand_value_str_ = std::to_string(rand_int) ; +} std::map DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& comm, const std::string& OutputDir) { + SetRandString() ; // sets rand_value_str_ used for naming things that might need a unique name + std::string saveToHDF5_str("MyStore"); if (! saveToDamarisHDF5_ ){ saveToHDF5_str = "#"; @@ -126,8 +142,9 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com } } #else - OpmLog::info(fmt::format("INFO: Opm::DamarisOutput::DamarisKeywords() : Paraview is not enabled in the Damaris library. " - "The commandline --damaris-python-paraview-script={} will be set to empty string", paraviewPythonFilename_)); + OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : Paraview is not enabled in the Damaris library. " + "The commandline --damaris-python-paraview-script={} will be set to empty string", + paraviewPythonFilename_)); paraviewPythonFilename_.clear(); #endif @@ -154,29 +171,40 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com } else { damarisOutputCollective_str = "FilePerCore"; } + OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : ", + damarisOutputCollective_str)); std::string simName_str; + // Check if simulation name was given on command line + // The simulation name is used as a prefix to name HDF5 files if (damarisSimName_.empty()) { - // Having a different simulation name is important if multiple simulations - // are running on the same node, as it is used to name the simulations shmem area - // and when one sim finishes it removes its shmem file. - // simName_str = damaris::Environment::GetMagicNumber(comm); - if (simName_str.empty()) { - // We will add a random value as GetMagicNumber(comm) requires Damaris v1.9.2 - // Seed with a real random value, if available - std::random_device r; - // Choose a random number between 0 and MAX_INT - std::default_random_engine e1(r()); - std::uniform_int_distribution uniform_dist(0, std::numeric_limits::max()); - int rand_int = uniform_dist(e1); - simName_str = "opm-flow-" + std::to_string(rand_int); - } else { - simName_str = "opm-flow-" + simName_str; - } + simName_str = "opm-flow-" + rand_value_str_; } else { simName_str = damarisSimName_; } + OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : 0) && (nDamarisNodes_ > 0)) { nDamarisNodes_ = 0; // Default is to use Damaris Cores @@ -194,20 +222,20 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com } else { nDamarisNodes_str = "0"; } + + OpmLog::info(fmt::format("Opm::DamarisOutput::DamarisKeywords() : damaris_keywords = { {"_SHMEM_BUFFER_BYTES_REGEX_", shmemSizeBytes_str}, {"_DC_REGEX_", nDamarisCores_str}, @@ -222,6 +250,7 @@ DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& com {"_PRESSURE_UNIT_","Pa"}, {"_MAKE_AVAILABLE_IN_PYTHON_",publishToPython_str}, /* must match + std::string damarisSimName_; // empty and set to "opm-flow-" if none provided on command line. Used as prefix to HDF5 filenames + std::string shmemName_; // empty and needs to be unique if multiple simulations are running on the same server/node. Used to name the Damaris shared memory region. std::string damarisLogLevel_ = "info"; std::string damarisDaskFile_ = ""; int nDamarisCores_ = 1; // this is the number of (Damaris server) cores per node int nDamarisNodes_ = 0; long shmemSizeBytes_ = 536870912; // 512 MB + + std::string rand_value_str_ ; // to be added to sheared memory name to make unique std::map getKeywords(const Parallel::Communication& comm, const std::string& OutputDir); + + void SetRandString(void); // sets the value of rand_value_str_ }; /** @@ -93,6 +98,7 @@ getDamarisKeywords(const Parallel::Communication& comm, const std::string& Outpu settings.nDamarisCores_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedCores); settings.nDamarisNodes_ = EWOMS_GET_PARAM(TypeTag, int, DamarisDedicatedNodes); settings.shmemSizeBytes_ = EWOMS_GET_PARAM(TypeTag, long, DamarisSharedMemorySizeBytes); + settings.shmemName_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisSharedMemoryName); settings.damarisLogLevel_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisLogLevel); settings.damarisDaskFile_ = EWOMS_GET_PARAM(TypeTag, std::string, DamarisDaskFile); return settings.getKeywords(comm, OutputDir); diff --git a/opm/simulators/utils/DamarisVar.hpp b/opm/simulators/utils/DamarisVar.hpp index f81c3cb2a..45b1d1c8b 100644 --- a/opm/simulators/utils/DamarisVar.hpp +++ b/opm/simulators/utils/DamarisVar.hpp @@ -161,38 +161,14 @@ public: }; // class DamarisVarBase /** - * class to store a Damaris variable representation for the XML file (can be - * used with /ref class DamarisKeywords). - * - * It is thought that the details stored in the object can be used to pass into - * an XML generation function e.g. DamarisKeywords - * - * Should be instantiated with something like the following: - * // The following needs to be defined in the Damaris XML file: - * // - * // - * // damaris::model::DamarisVar dam_var = new - * damaris::model::DamarisVar(1, {std::string("n_connectivity_ph")}, - * std::string("topologies/topo/elements/connectivity"), rank_); - * dam_var->SetDamarisParameterAndShmem( { geomData.getNCorners() } ); - * - * int * shmem_mpi_ptr = dam_var->data_ptr(); - * // Fill the created memory area - * for (int i = 0; i <; i++ ) - * { - * shmem_mpi_ptr[i] = rank_; - * } - * delete dam_var; // this tells Damaris that the shared memory that it - * supplied is at its disposal. It will print error messages too. - * - */ +* class to store a Damaris variable representation for the XML file +* (can be used with /ref class DamarisKeywords). +* +* It is thought that the details stored in the object can be used to pass +* into an XML generation function e.g. DamarisKeywords +* +*/ template class DamarisVar : public DamarisVarBase { - int dims_; int num_params_; //!< Each paramater name string will need a value and they //!< are set in SetDamarisParameter() std::vector @@ -226,55 +202,67 @@ template class DamarisVar : public DamarisVarBase { size_t current_size_; //!< The total number of elements that may be held by this //!< part of the variable - returned by the size() method. + //!< N.B. the actual size of the data area is dependent on + //!< how the XML is written, as paramaters can + //!< be augmented by basic maths relationships. This value + //!< may not even be initialised if ParameterIsSet() method + //!< is being used (e.g. in version 2/ of the constructor below). public: - /** - * Constructor - sets private data values and dos not initialise the shared memory area. - * - * Two usages: - * Example XML definition: - * - * - * - * - * - * 1/ The variable's layout needs to be initialised via parameters : - * // Create the DamarisVar object: - * damaris::model::DamarisVar* MYVARNAME_2d = new damaris::model::DamarisVar(2, - * {std::string("my_param_name1"), std::string("my_param_name2")}, - * {100, 25}, - * std::string("MYVARNAME"), rank_); - * // sets the paramater sizes (so, here, my_param_name1 == 25 and my_param_name2 == 100) - * MYVARNAME_2d->SetDamarisParameterAndShmem( {25, 100 } }; - * // Get a pointer to the memeory and use it - * T * mymemory = MYVARNAME_2d->data(); - * ... write data to mymemory .... - * delete MYVARNAME_2d; - * or, - * 2/ The variable's layout has been initialised via parameters in another variable - * (i.e. "my_param_name1" and "my_param_name2" have been previously set in the code) - * // Create the DamarisVar object: - * damaris::model::DamarisVar* MYVARNAME_2d = new damaris::model::DamarisVar(2, - * {std::string("my_param_name1"), std::string("my_param_name2")}, - * std::string("MYVARNAME"), rank_); - * // explicitly state that the paramater values have been set somewhere else in the code previously. - * MYVARNAME_2d->ParameterIsSet(); - * MYVARNAME_2d->SetPointersToDamarisShmem() - * // Get a pointer to the memeory and use it - * T * mymemory = MYVARNAME_2d->data(); - * ... write data to mymemory .... - * delete MYVARNAME_2d; - * - * /param [IN] dims Used to check that the inputs to SetDamarisPosition() - * have the same number of values - one value for each dimension - * /param [IN] param_names The name the Damaris paramaters. These names (in typical use) control - * a Damaris variables size (names are defined in the Damaris XML file). - * /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file) - * /param [IN] rank The rank of the process. Used for error output. - */ +/** +* Constructor - sets private data values and dos not initialise the shared memory area. +* +* N.B. These objects need a matching and in the Damaris XML file +* +* Two usages: +* Example XML definition: +* +* +* +* +* +* 1/ The variable's layout needs to be initialised via parameters : +* // Create the DamarisVar object: +* damaris::model::DamarisVar MYVARNAME_2d(2,{std::string("my_param_name1"), +* std::string("my_param_name2")}, +* {100, 25}, +* std::string("MYVARNAME"), rank_); +* // sets the paramater sizes (so, here, my_param_name1 == 25 and my_param_name2 == 100) +* MYVARNAME_2d.SetDamarisParameterAndShmem( {25, 100 } }; +* // Get a pointer to the memeory and use it +* T * mymemory = MYVARNAME_2d.data(); +* ... write data to mymemory .... +* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope. +* or, +* 2/ The variable's layout has been initialised via parameters in another variable +* (i.e. "my_param_name1" and "my_param_name2" have been previously set in the code) +* // Create the DamarisVar object: +* damaris::model::DamarisVar MYVARNAME_2d(2, {std::string("my_param_name1"), + std::string("my_param_name2")}, +* std::string("MYVARNAME"), rank_); +* +* // explicitly state that the paramater values have been set somewhere else in the code previously. +* MYVARNAME_2d.ParameterIsSet(); +* +* N.B. This will not set the internal current_size_ value so the size() value will +* not be correct <- This is important to remember +* +* MYVARNAME_2d.SetPointersToDamarisShmem() +* // Get a pointer to the memeory and use it +* T * mymemory = MYVARNAME_2d.data(); +* ... write data to mymemory .... +* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope. +* +* /param [IN] dims Used to check that the inputs to SetDamarisPosition() +* have the same number of values - one value for each dimension +* /param [IN] param_names The name the Damaris paramaters. These names (in typical use) control +* a Damaris variables size (names are defined in the Damaris XML file). +* /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file) +* /param [IN] rank The rank of the process. Used for error output. +*/ DamarisVar(int dims, std::vector param_names, std::string variable_name, int rank) - : dims_(dims), param_names_(param_names), variable_name_(variable_name), + : param_names_(param_names), variable_name_(variable_name), rank_(rank) { dam_err_ = DAMARIS_OK; @@ -300,38 +288,40 @@ public: has_error_ = false; } - /** - * Constructor - Sets private data values and also initialises the Damaris shared memory area for writing (and reading) - * by specifying the values for the variables parameters . i.e. makes the data_ptr() - * - * Example use: - * Example XML definition: - * - * - * - * - * // The paramaters are intialized in the constructor code - * damaris::model::DamarisVar* MYVARNAME_2d = new damaris::model::DamarisVar(2, - * {std::string("my_param_name1"), std::string("my_param_name2")}, - * {100, 25}, - * std::string("MYVARNAME"), rank_); - * T * mymemory = MYVARNAME_2d->data(); - * ... write data to mymemory .... - * delete MYVARNAME_2d; - * - * /param [IN] dims Used to check that the inputs to SetDamarisPosition() have - * the same number of values - one value for each dimension - * /param [IN] param_names The name the Damaris paramaters. These names (in typical use) - * control a Damaris variables size (names are defined in the Damaris XML file). - * /param [IN] param_values The values of the paramaters - this defines how much memory we will - * have access to in the shared memory area (on the current and ongoing iterations, - * until later modified to new values) - * /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file) - * /param [IN] rank The rank of the process. Used for error output. - */ +/** +* Constructor - Sets private data values and also initialises the Damaris shared memory area for writing (and reading) +* by specifying the values for the variables parameters . +* i.e. makes the data() pointer available and sets the size of the memory block it points to. +* +* N.B. These objects need a matching and in the Damaris XML file +* +* Example use: +* Example XML definition: +* +* +* +* +* // The paramaters are intialized in the constructor code +* damaris::model::DamarisVar MYVARNAME_2d(2,{std::string("my_param_name1"), std::string("my_param_name2")}, +* {100, 25}, +* std::string("MYVARNAME"), rank_); +* T * mymemory = MYVARNAME_2d.data(); +* ... write data to mymemory .... +* // Damaris shared memory is tidied up when object MYVARNAME_2d is out of scope. +* +* /param [IN] dims Used to check that the inputs to SetDamarisPosition() have +* the same number of values - one value for each dimension +* /param [IN] param_names The name the Damaris paramaters. These names (in typical use) +* control a Damaris variables size (names are defined in the Damaris XML file). +* /param [IN] param_values The values of the paramaters - this defines how much memory we will +* have access to in the shared memory area (on the current and ongoing iterations, +* until later modified to new values) +* /param [IN] variable_name The name of the Damaris variable (defined in the Damaris XML file) +* /param [IN] rank The rank of the process. Used for error output. +*/ DamarisVar(int dims, std::vector param_names, std::vector param_values, std::string variable_name, int rank) - : dims_(dims), param_names_(param_names), variable_name_(variable_name), + : param_names_(param_names), variable_name_(variable_name), rank_(rank) { DamarisVar(dims, param_names, variable_name, rank); setDamarisParameterAndShmem( @@ -589,9 +579,9 @@ public: * variable_name_ */ void setDamarisPosition(const std::vector &positionsVals) { - assert(positionsVals.size() == dims_); + assert(positionsVals.size() == num_params_); - for (int pos_dim = 0; pos_dim < dims_; pos_dim++) { + for (int pos_dim = 0; pos_dim < num_params_; pos_dim++) { positions_[pos_dim] = positionsVals[pos_dim]; } dam_err_ = diff --git a/opm/simulators/utils/GridDataOutput.hpp b/opm/simulators/utils/GridDataOutput.hpp index 35f00223f..dc0dfd7c3 100644 --- a/opm/simulators/utils/GridDataOutput.hpp +++ b/opm/simulators/utils/GridDataOutput.hpp @@ -48,7 +48,7 @@ // N.B. does not seem to be able to be allocated with new operator. Opm::GridDataOutput::SimMeshDataAccessor geomData(gridView, - Dune::Partition::interior ); + Dune::Partition::interior ); geomData.printGridDetails(); @@ -58,7 +58,7 @@ double * x_vert = new double[nvert]; double * y_vert = new double[nvert]; double * z_vert = new double[nvert]; - geomData.writeGridPoints(x_vert,y_vert,z_vert); + geomData.writeGridPoints(x_vert,y_vert,z_vert, nvert); ... do something with vertex data x_vert, y_vert and z_vert .... @@ -68,11 +68,18 @@ // example using AOS double * xyz_vert_aos = new double[nvert*3]; - geomData.writeGridPoints_AOS(xyz_vert_aos); + geomData.writeGridPoints_AOS(xyz_vert_aos, nvert); ... do something with vertex data xyz_vert_aos.... delete [] xyz_vert_aos; + + + // example using SOA with std::vector + std::vector xyz_vert_soa(nvert*3); + geomData.writeGridPoints_SOA(xyz_vert_soa); + + ... do something with vertex data xyz_vert_soa.... */ @@ -148,8 +155,8 @@ public: Count the vertices, cells and corners. Count all the vertices ( the Dune::Partitions::all partition ) as then we - do not need to renumber the vertices as all the subsets use references to - the full set. + do not need to renumber the vertices as all the subsets use references to + the full set. */ void countEntities() { // We include all the vertices for this ranks partition @@ -166,10 +173,22 @@ public: } } + + /** + Write the positions of vertices - directly to the pointers given in + parameters + + @param x_inout to be filled with x coordinate verticies + @param y_inout to be filled with y coordinate verticies + @param y_inout to be filled with z coordinate verticies + @param max_size the maximum number of elements of type T that can be + written to the input pointer memory regions. + + Returns the number of vertices written + */ template long writeGridPoints(T *x_inout, T *y_inout, T *z_inout, long max_size = 0) { if (max_size < nvertices_) { - // assert(max_size >= nvertices_); OPM_THROW(std::runtime_error, "Opm::GridDataOutput::writeGridPoints( T& x_inout, T& " "y_inout, T& z_inout ) " + @@ -207,21 +226,32 @@ public: /** Write the positions of vertices - directly to the pointers given in parameters - + + @param x_inout to be filled with x coordinate verticies + @param y_inout to be filled with y coordinate verticies + @param y_inout to be filled with z coordinate verticies + + All parameters must have a size() and data() method (e.g. a std::vector) + and the current size() must be big enough + Returns the number of vertices written */ - template - long writeGridPoints(T &x_inout, T &y_inout, T &z_inout) { - size_t check_size = x_inout.size(); - + template + long writeGridPoints(VectType &x_inout, VectType &y_inout, VectType &z_inout) { + size_t check_size_x = x_inout.size(); + size_t check_size_y = y_inout.size(); + size_t check_size_z = z_inout.size(); + using VT = decltype(x_inout.data()[0]); - if (check_size < nvertices_) { + if ((check_size_x < nvertices_) || (check_size_y < nvertices_) || (check_size_z < nvertices_)){ // assert(check_size >= nvertices_); OPM_THROW(std::runtime_error, - "Opm::GridDataOutput::writeGridPoints( T& x_inout, T& " - "y_inout, T& z_inout ) " + - " Input objects size " + std::to_string(check_size) + + "Opm::GridDataOutput::writeGridPoints( VectType& x_inout, VectType& " + "y_inout, VectType& z_inout ) At least one of the inputs" + + " object x size " + std::to_string(check_size_x) + + " object y size " + std::to_string(check_size_y) + + " object z size " + std::to_string(check_size_z) + " is not sufficient to fit the nvertices_ values( " + std::to_string(nvertices_) + " )"); } @@ -254,10 +284,15 @@ public: } /** - Write positions of vertices as array of structures : x,y,z,x,y,z,x,y,z,... - - Returns the number of vertices written - */ + Write the positions of vertices - directly to the pointers given in + parameters as Array of Structures x,y,z,x,y,z,x,y,z,... + + @param xyz_inout is the array to be filled with x,y,z coordinate verticies. + @param max_size is the maximum number x,y,z structures with elements of type T + that can be written to the input pointer memory regions. + + Returns the number of vertices written + */ template long writeGridPoints_AOS(T *xyz_inout, long max_size = 0) { if (max_size < nvertices_ * 3) { @@ -288,10 +323,15 @@ public: return ((i) / 3); } + /** - Write positions of vertices as array of structures : x,y,z,x,y,z,x,y,z,... - - Returns the number of vertices written + Write the positions of vertices - directly to the pointers given in + parameters as Array of Structures x,y,z,x,y,z,x,y,z,... + + @param xyz_inout is the array to be filled with x,y,z coordinate verticies. + The object VectType must have a size() and data() method (e.g. a std::vector) + + Returns the number of vertices written */ template long writeGridPoints_AOS(VectType &xyz_inout) { size_t check_size = xyz_inout.size(); @@ -328,12 +368,16 @@ public: return ((i) / 3); } - /** - Write positions of vertices as structure of arrays : - x,x,x,...,y,y,y,...,z,z,z,... - - Returns the number of vertices written - */ +/** + Write the positions of vertices - directly to the pointers given in + parameters as Structure of Arrays: x,x,x,...,y,y,y,...,z,z,z,... + + @param xyz_inout is the array to be filled with x,y,z coordinate verticies. + @param max_size number of verticies (x,...y,...z,... structures) with elements of type T + that can be written to the input pointer memory regions. + + Returns the number of vertices written + */ template long writeGridPoints_SOA(T *xyz_inout, long max_size = 0) { if (max_size < nvertices_ * 3) { @@ -371,11 +415,14 @@ public: } /** - Write positions of vertices as structure of arrays : - x,x,x,...,y,y,y,...,z,z,z,... - - Returns the number of vertices written - */ + Write the positions of vertices - directly to the pointers given in + parameters as Structure of Arrays: x,x,x,...,y,y,y,...,z,z,z,... + + @param xyz_inout is the array to be filled with x,y,z coordinate verticies. + The object VectType must have a size() and data() method (e.g. a std::vector) + + Returns the number of vertices written + */ template long writeGridPoints_SOA(VectType &xyz_inout) { size_t check_size = xyz_inout.size(); @@ -420,7 +467,12 @@ public: /** * Write the connectivity array - directly to the pointer given in parameter 1 Reorders the indices as selected either in DUNE order or VTK order. - + + @param connectivity_inout is the array to be filled with connectivity indexes + (i.e. the index into the vertex array) + @param whichOrder, is the order that verticies are traversed to create a cell (VTK or DUNE) + @param max_size is used to check that the space available in the input pointer + parameter will fit the number of corner values written. Returns the number of corner indices written. */ template @@ -465,11 +517,17 @@ public: } /** - * Write the connectivity array - directly to the pointer given in parameter 1 - Reorders the indices as selected either in DUNE order or VTK order. - - Returns the number of corner indices written. - */ + * Write the connectivity array - directly to a VectType object given in parameter 1 + Reorders the indices as selected either in DUNE order or VTK order. + + @param connectivity_inout is the array to be filled with connectivity indexes + (i.e. the index into the vertex array) + The object VectType must have a size() and data() method (e.g. a std::vector) + @param whichOrder, is the order that verticies are traversed to create a cell (VTK or DUNE) + @param max_size is used to check that the space available in the input pointer + parameter will fit the number of corner values written. + Returns the number of corner indices written. + */ template long writeConnectivity(VectType &connectivity_inout, ConnectivityVertexOrder whichOrder) { @@ -514,10 +572,18 @@ public: return (i); } - /** + + /** * Write the offsets values - directly to the pointer given in parameter 1 - Returns the (number of offset values written + 1) - */ + + @param offsets_inout is the array to be filled with offsets into the connectivity array + (i.e. the index into the connectivity array to determine the vertices used for + the particular cell) + @param max_size is used to check that the space available in the input pointer + parameter will fit the number of cell offset values written. + + Returns number of offset values written + 1 +*/ template long writeOffsetsCells(Integer *offsets_inout, long max_size = 0) { if (max_size < ncells_) { @@ -539,10 +605,16 @@ public: return (i); // This should be 1 greater than ncells_ } - /** - * Write the offsets values - directly to the pointer given in parameter 1 - Returns the (number of offset values written + 1) - */ +/** + * Write the offsets values - directly to a VectType object given in parameter 1 + + @param offsets_inout is the array to be filled with offsets into the connectivity array + (i.e. the index into the connectivity array to determine the vertices used for + the particular cell). + The object VectType must have a size() and data() method (e.g. a std::vector) + + Returns number of offset values written + 1 +*/ template long writeOffsetsCells(VectType &offsets_inout) { size_t check_size = offsets_inout.size(); if (check_size < ncells_) { @@ -567,9 +639,16 @@ public: return (i); // This should be 1 greater than ncells_ } - /** - * Write the Cell types array - directly to the pointer given in parameter 1 - */ +/** + * Write the cell types values - directly to the pointer given in parameter 1 + + @param types_inout is the array to be filled with the cell types (VTK defined values) + + @param max_size is used to check that the space available in the input pointer + parameter will fit the number of cell offset values written. + + Returns number of cells type values written +*/ template long writeCellTypes(Integer *types_inout, long max_size = 0) { if (max_size < ncells_) { @@ -589,14 +668,18 @@ public: return (i); } - /** - * Write the Cell types array - directly to the pointer given in parameter 1 - */ +/** + * Write the cell types values - directly to the VectType object given in parameter 1 + + @param types_inout is the array to be filled with the cell types (VTK defined values) + The object VectType must have a size() and data() method (e.g. a std::vector) + + Returns number of cells type values written +*/ template long writeCellTypes(VectType &types_inout) { size_t check_size = types_inout.size(); if (check_size < ncells_) { - // assert(check_size >= ncells_); OPM_THROW( std::runtime_error, "Opm::GridDataOutput::writeCellTypes( VectType& types_inout ) " + diff --git a/opm/simulators/utils/initDamarisXmlFile.cpp b/opm/simulators/utils/initDamarisXmlFile.cpp index fbc6e11a0..6830bd04c 100644 --- a/opm/simulators/utils/initDamarisXmlFile.cpp +++ b/opm/simulators/utils/initDamarisXmlFile.cpp @@ -39,7 +39,7 @@ std::string initDamarisXmlFile() - +