added ability to pass through multiple variables to Damaris, with full list still defined in initDamarisXmlFile.cpp ; Also adds ability to limit the variables to be passed through using --damaris-limit-variables=<CSV list> coammand line option

This commit is contained in:
josh bowden 2024-05-14 21:08:47 +02:00
parent ae61361061
commit 3630177e3e
7 changed files with 200 additions and 77 deletions

View File

@ -87,7 +87,10 @@ template<class TypeTag, class MyTypeTag>
struct DamarisDaskFile {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct DamarisLimitVariables {
using type = UndefinedProperty;
};
} // namespace Opm::Properties
#endif // OPM_DAMARIS_PROPERTIES_HPP

View File

@ -42,7 +42,7 @@ int setPosition(const char* field, int rank, int64_t pos)
{
int dam_err = damaris_set_position(field, &pos);
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{})"
OpmLog::error(fmt::format("damariswriter::setPosition() : ( rank:{}) "
"damaris_set_position({}, ...), Damaris Error: {} ",
rank, field, damaris_error_string(dam_err)));
}
@ -54,7 +54,7 @@ int setParameter(const char* field, int rank, int value)
{
int dam_err = damaris_parameter_set(field, &value, sizeof(int));
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("(rank:{}) Damaris library produced an error result for "
OpmLog::error(fmt::format("damariswriter::setParameter() (rank:{}) "
"damaris_parameter_set(\"{}\",...)", rank, field));
}
@ -65,7 +65,7 @@ int write(const char* field, int rank, const void* data)
{
int dam_err = damaris_write(field, data);
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
OpmLog::error(fmt::format("damariswriter::write() : ( rank:{}) "
"damaris_write({}, ...), Damaris Error: {} ",
rank, field, damaris_error_string(dam_err)));
}
@ -77,7 +77,7 @@ int endIteration(int rank)
{
int dam_err = damaris_end_iteration();
if (dam_err != DAMARIS_OK) {
OpmLog::error(fmt::format("damariswriter::writeOutput() : ( rank:{}) "
OpmLog::error(fmt::format("damariswriter::endIteration() : ( rank:{}) "
"damaris_end_iteration(), Damaris Error: {} ",
rank, damaris_error_string(dam_err)));
}
@ -130,17 +130,9 @@ int setupWritingPars(Parallel::Communication comm,
OPM_THROW(std::runtime_error, "setupDamarisWritingPars() n_elements_global_max "
"> std::numeric_limits<int>::max() " + std::to_string(dam_err));
}
// Use damaris_set_position to set the offset in the global size of the array.
// This is used so that output functionality (e.g. HDF5Store) knows global offsets of the data of the ranks
setPosition("PRESSURE", comm.rank(), elements_rank_offsets[comm.rank()]);
dam_err = setPosition("GLOBAL_CELL_INDEX", comm.rank(), elements_rank_offsets[comm.rank()]);
// Set the size of the MPI variable
DamarisVar<int> mpi_rank_var(1, {"n_elements_local"}, "MPI_RANK", comm.rank());
mpi_rank_var.setDamarisPosition({static_cast<int64_t>(elements_rank_offsets[comm.rank()])});
return dam_err;
}
}

View File

@ -42,6 +42,7 @@
#include <opm/simulators/flow/FlowBaseVanguard.hpp>
#include <opm/simulators/flow/OutputBlackoilModule.hpp>
#include <opm/simulators/utils/DamarisVar.hpp>
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <opm/simulators/utils/DeferredLoggingErrorHelpers.hpp>
#include <opm/simulators/utils/GridDataOutput.hpp>
#include <opm/simulators/utils/ParallelSerialization.hpp>
@ -53,6 +54,8 @@
#include <numeric>
#include <string>
#include <vector>
#include <unordered_set>
namespace Opm {
@ -154,6 +157,11 @@ public:
"This name should be unique if multiple simulations are running on "
"the same node/server as it is used for the Damaris shmem name and by "
"the Python Dask library to locate sections of variables.");
Parameters::registerParam<TypeTag, Properties::DamarisLimitVariables>
("A comma separated list of variable names that a user wants to pass"
"through via DamarisOutput::DamarisWriter::writeOutput)() to the "
"damaris_write() call. This can be used to limit the number of"
"variables being passed to the Daamis plugins (Paraview, Python and HDF5)");
}
// The Simulator object should preferably have been const - the
@ -210,6 +218,8 @@ public:
this->damarisOutputModule_ = std::make_unique<OutputBlackOilModule<TypeTag>>
(simulator, this->eclIO_->finalSummaryConfig(), this->collectOnIORank_);
}
wanted_vars_set_ = Opm::DamarisOutput::getSetOfIncludedVariables<TypeTag>();
}
/*!
@ -247,7 +257,7 @@ public:
dam_err_ = DamarisOutput::setupWritingPars(simulator_.vanguard().grid().comm(),
numElements_, elements_rank_offsets_);
// sets data for non-time-varying variables MPI_RANK and GLOBAL_CELL_INDEX
// sets positions and data for non-time-varying variables MPI_RANK and GLOBAL_CELL_INDEX
this->setGlobalIndexForDamaris() ;
// Set the geometry data for the mesh model.
@ -259,7 +269,7 @@ public:
this->damarisUpdate_ = false;
}
if (this->damarisOutputModule_->getPRESSURE_ptr() != nullptr)
/*if (this->damarisOutputModule_->getPRESSURE_ptr() != nullptr)
{
dam_err_ = DamarisOutput::setPosition("PRESSURE", rank_,
this->elements_rank_offsets_[rank_]);
@ -267,7 +277,73 @@ public:
this->damarisOutputModule_->getPRESSURE_ptr());
dam_err_ = DamarisOutput::endIteration(rank_);
}*/
// Call damaris_set_position() for all available variables
// There is an assumption that all variables are the same size, with the same offset.
// see initDamarisTemplateXmlFile.cpp for the Damaris XML descriptions.
for ( auto damVar : localCellData ) {
// std::map<std::string, data::CellData>
const std::string name = damVar.first ;
dam_err_ = DamarisOutput::setPosition(name.c_str(), rank_,
this->elements_rank_offsets_[rank_]);
}
// Call damaris_write() for all available variables
for ( auto damVar : localCellData )
{
// std::map<std::string, data::CellData>
const std::string& name = damVar.first ;
std::unordered_set<std::string>::const_iterator is_in_set = wanted_vars_set_.find ( name );
if ((is_in_set != wanted_vars_set_.end() ) || (wanted_vars_set_.size() == 0)) {
data::CellData dataCol = damVar.second ;
OpmLog::debug(fmt::format("Name of Damaris Variable : ( rank:{}) name: {} ", rank_, name));
// It does not seem I can test for what type of data is present (double or int)
// in the std::variant within the data::CellData, so I will use a try catch block.
// Although, only MPI_RANK and GLOBAL_CELL_INDEX are set as integer types (in the
// XML file) so it is a moot point.
// We could use damaris_get_type() to check what the type is specified as
// within the Damaris XML file
try {
if (dataCol.data<double>().size() >= this->numElements_) {
dam_err_ = DamarisOutput::write(name.c_str(), rank_,
dataCol.data<double>().data()) ;
}
}
catch (std::bad_variant_access const& ex) {
// Not a std::vector<double>
if (dataCol.data<int>().size() >= this->numElements_) {
dam_err_ = DamarisOutput::write(name.c_str(), rank_,
dataCol.data<int>().data()) ;
}
}
}
}
/*
Code for when we want to pass to Damaris the single cell 'block' data variables
auto mybloc = damarisOutputModule_->getBlockData() ;
for ( auto damVar : mybloc ) {
// std::map<std::string, data::CellData>
const std::string name = std::get<0>(damVar.first) ;
const int part = std::get<1>(damVar.first) ;
double dataCol = damVar.second ;
std::cout << "Name of Damaris Block Varaiable : (" << rank_ << ") " << name << " part : " << part << " Value : " << dataCol << std::endl ;
}
dam_err_ = DamarisOutput::endIteration(rank_);
*/
if (this->damarisOutputModule_->getPRESSURE_ptr() != nullptr)
{
dam_err_ = DamarisOutput::endIteration(rank_);
}
} // end of ! isSubstep
}
@ -276,6 +352,7 @@ private:
int rank_ ;
int nranks_ ;
int numElements_ ; ///< size of the unique vector elements
std::unordered_set<std::string> wanted_vars_set_ ;
Simulator& simulator_;
std::unique_ptr<OutputBlackOilModule<TypeTag>> damarisOutputModule_;
@ -289,6 +366,16 @@ private:
void setGlobalIndexForDamaris ()
{
// Use damaris_set_position to set the offset in the global size of the array.
// This is used so that output functionality (e.g. HDF5Store) knows global offsets of the data of the ranks
// setPosition("PRESSURE", comm.rank(), elements_rank_offsets_[comm.rank()]);
dam_err_ = DamarisOutput::setPosition("GLOBAL_CELL_INDEX", rank_, elements_rank_offsets_[rank_]);
// Set the size of the MPI variable
// N.B. MPI_RANK is only saved to HDF5 if --damaris-save-mesh-to-hdf=true is specified
DamarisVarInt mpi_rank_var(1, {"n_elements_local"}, "MPI_RANK", rank_);
mpi_rank_var.setDamarisPosition({static_cast<int64_t>(elements_rank_offsets_[rank_])});
// GLOBAL_CELL_INDEX is used to reorder variable data when writing to disk
// This is enabled using select-file="GLOBAL_CELL_INDEX" in the <variable> XML tag
if (this->collectOnIORank_.isParallel()) {
@ -307,10 +394,12 @@ private:
// We will add the MPI rank value directly into shared memory using the DamarisVar
// wrapper of the C based Damaris API.
// The shared memory is given back to Damaris when the DamarisVarInt goes out of scope.
DamarisVarInt mpi_rank_var_test(1, {"n_elements_local"}, "MPI_RANK", rank_);
mpi_rank_var_test.setDamarisParameterAndShmem( {this->numElements_ } ) ;
// DamarisVarInt mpi_rank_var_test(1, {"n_elements_local"}, "MPI_RANK", rank_);
mpi_rank_var.setDamarisParameterAndShmem( {this->numElements_ } ) ;
// Fill the created memory area
std::fill(mpi_rank_var_test.data(), mpi_rank_var_test.data() + numElements_, rank_);
std::fill(mpi_rank_var.data(), mpi_rank_var.data() + numElements_, rank_);
}
void writeDamarisGridOutput()
@ -374,7 +463,7 @@ private:
"topologies/topo/elements/types", rank_) ;
var_types.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
// Copy the mesh data from the Durne grid
// Copy the mesh data from the Dune grid
long i = 0 ;
GridDataOutput::ConnectivityVertexOrder vtkorder = GridDataOutput::VTK ;

View File

@ -387,6 +387,11 @@ template<class TypeTag>
struct DamarisDaskFile<TypeTag, TTag::FlowBaseProblem> {
static constexpr auto value = "";
};
// Specify the the exact variables to be passed through to Damaris (must exist in the XML file / intiDamarisXmlFile.cpp)
template<class TypeTag>
struct DamarisLimitVariables<TypeTag, TTag::FlowBaseProblem> {
static constexpr auto value = "";
};
#endif
// If available, write the ECL output in a non-blocking manner
template<class TypeTag>

View File

@ -95,9 +95,10 @@ void DamarisSettings::SetRandString(void)
std::uniform_int_distribution<int> uniform_dist(0, std::numeric_limits<int>::max());
int rand_int = uniform_dist(e1);
rand_value_str_ = std::to_string(rand_int) ;
rand_value_str_ = std::to_string(rand_int) ;
}
std::map<std::string, std::string>
DamarisSettings::getKeywords([[maybe_unused]] const Parallel::Communication& comm,
const std::string& OutputDir)

View File

@ -25,6 +25,9 @@
#include <map>
#include <string>
#include <sstream>
#include <algorithm>
#include <unordered_set>
/*
Below is the std::map with the keywords that are supported by Damaris.
@ -59,6 +62,7 @@ struct DamarisSettings {
std::string shmemName_; // empty and needs to be unique if multiple simulations are running on the same server/node. Used to name the Damaris shared memory region.
std::string damarisLogLevel_ = "info";
std::string damarisDaskFile_ = "";
// std::string damarisLimitVars_ = "";
int nDamarisCores_ = 1; // this is the number of (Damaris server) cores per node
int nDamarisNodes_ = 0;
long shmemSizeBytes_ = 536870912; // 512 MB
@ -68,6 +72,8 @@ struct DamarisSettings {
std::map<std::string, std::string>
getKeywords(const Parallel::Communication& comm,
const std::string& OutputDir);
void SetRandString(void); // sets the value of rand_value_str_
};
@ -104,6 +110,27 @@ getDamarisKeywords(const Parallel::Communication& comm, const std::string& Outpu
return settings.getKeywords(comm, OutputDir);
}
template<class TypeTag>
std::unordered_set<std::string>
getSetOfIncludedVariables(void)
{
std::unordered_set<std::string> resuset ;
std::string tstr;
std::string damarisLimitVars = Parameters::get<TypeTag, Properties::DamarisLimitVariables>();
std::stringstream ss(damarisLimitVars);
// Use while loop to check the getline() function condition.
while (std::getline(ss, tstr, ',')) {
//remove whitespace
// tstr.erase(std::remove_if(tstr.begin(), tstr.end(), ::isspace), tstr.end());
std::string::iterator end_pos = std::remove(tstr.begin(), tstr.end(), ' ');
tstr.erase(end_pos, tstr.end());
// place in set (no duplicates possible in set)
resuset.insert(tstr) ;
}
return (resuset) ;
}
} // namespace Opm::DamarisOutput

View File

@ -54,65 +54,71 @@ std::string initDamarisXmlFile()
<layout name="zonal_layout_usmesh" type="double" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="PRESSURE" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="_PRESSURE_UNIT_" centering="zonal" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="MY_TEST_VAR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
_MORE_VARIABLES_REGEX_
<variable name="MPI_RANK" layout="zonal_layout_usmesh_integer" type="scalar" visualizable="true" mesh="us_mesh" unit="rank" centering="zonal" store="_MYSTORE_MESH_OR_EMPTY_REGEX_" time-varying="false" select-file="GLOBAL_CELL_INDEX" script="_MAKE_AVAILABLE_IN_PYTHON_" comment="The MPI rank of each cell"/>
<variable name="KRNSW_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="KRNSW_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="PCSWM_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="PCSWM_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="PPCW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="RS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" comment="Dissolved Gas units Gas Oil Ratio" />
<variable name="RV" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SOMAX" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GASKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GAS_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GAS_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OILKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIL_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIL_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WATKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WAT_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WAT_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="FIPGAS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="FIPOIL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="FIPWAT" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="KRNSW_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="KRNSW_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="PCSWM_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="PCSWM_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="PPCW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="RS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" comment="Dissolved Gas units Gas Oil Ratio" />
<variable name="RV" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="SOMAX" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="1OVERBG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="1OVERBO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="1OVERBW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GASKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GAS_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GAS_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="OILKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="OIL_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="OIL_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="WATKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="WAT_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="WAT_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="2FBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="4FBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="DFBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GCDI" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GCDM" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GKDI" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GKDM" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GKMO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GKTR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GMDS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GMGP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GMIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GMMO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GMTR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="HTOF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="RPV" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="S36F" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SEAF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SGAS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SWAT" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="TFBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WCD" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="2FBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="4FBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="DFBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GCDI" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GCDM" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GKDI" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GKDM" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GKMO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GKTR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GMDS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GMGP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GMIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GMMO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="GMTR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="HTOF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="OIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="OIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="OIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="OIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="RPV" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="S36F" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="SEAF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="SGAS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="SIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="SWAT" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="TFBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="WCD" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="WIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="WIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="WIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<variable name="WIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="us_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_MAKE_AVAILABLE_IN_PYTHON_" />
<parameter name="n_coords_local" type="int" value="1" />
<layout name="n_coords_layout" type="double" dimensions="n_coords_local" comment="For the individual x, y and z coordinates of the mesh vertices, these values are referenced in the topologies/topo/subelements/connectivity_pg data" />