OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Move Damaris functionality to it's own class.

Browse Source 
Step one for moving Damaris calls out of EclWriter class and into its own DamarisWriter class;
EclProblem now calls both writeOutput methods and passes in the data::Solution object;

Add fix for first writeOutput() call not having PRESSURE data available;

data::Solution is now passed by rvalue ref into eclWriter::writeOutput();

guard added to prevent inclusion of damariswriter.hh
This commit is contained in:
josh bowden
2023-09-09 23:02:09 +02:00
parent 1d33e7caf0
commit 1e2d9360d7
9 changed files with 561 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
opm/simulators/utils/DamarisKeywords.cpp
View File

@@ -44,6 +44,10 @@ DamarisKeywords(std::string OutputDir, bool enableDamarisOutputCollective)
{"_MORE_VARIABLES_REGEX_", ""},
{"_PATH_REGEX_", OutputDir},
{"_MYSTORE_OR_EMPTY_REGEX_", "MyStore"},
{"_PYTHON_OR_EMPTY_REGEX_", ""},
{"_PYTHON_XML_NAME_", "#"},
{"_PARAVIEWPY_OR_EMPTY_REGEX_", ""},
{"_MESHNAME_OR_HASH_", "#"},
};
return damaris_keywords;
} else {
@@ -55,6 +59,10 @@ DamarisKeywords(std::string OutputDir, bool enableDamarisOutputCollective)
{"_MORE_VARIABLES_REGEX_", ""},
{"_PATH_REGEX_", OutputDir},
{"_MYSTORE_OR_EMPTY_REGEX_", "MyStore"},
{"_PYTHON_OR_EMPTY_REGEX_", ""},
{"_PYTHON_XML_NAME_", "#"},
{"_PARAVIEWPY_OR_EMPTY_REGEX_", ""},
{"_MESHNAME_OR_HASH_", "#"},
};
return damaris_keywords;
}

69
opm/simulators/utils/DamarisOutputModule.cpp
View File

@@ -1,5 +1,6 @@
/*
Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
Copyright 2023 INRIA
This file is part of the Open Porous Media project (OPM).
@@ -75,72 +76,4 @@ initializeDamaris(MPI_Comm comm, int mpiRank, std::string outputDir, bool enable
}
}
void
setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid)
{
int damaris_err = DAMARIS_OK;
const int nranks = comm.size();
const int rank = comm.rank();
std::vector<unsigned long long> elements_rank_sizes(nranks); // one for each rank -- to be gathered from each client rank
std::vector<unsigned long long> elements_rank_offsets(nranks); // one for each rank, first one 0 -- to be computed - Probably could use MPI_Scan()!
// n_elements_local_grid should be the full model size
const unsigned long long n_elements_local = n_elements_local_grid;
// Left in for debugging, but commented out to avoid spamming the terminal from non-output ranks.
// std::cout << "INFO (" << rank << "): n_elements_local_grid = " << n_elements_local_grid << std::endl;
// This gets the n_elements_local from all ranks and copies them to a std::vector of all the values on all ranks
// (elements_rank_sizes[]).
comm.allgather(&n_elements_local, 1, elements_rank_sizes.data());
elements_rank_offsets[0] = 0ULL;
// This scan makes the offsets to the start of each ranks grid section if each local grid data was concatenated (in
// rank order)
for (int t1 = 1; t1 < nranks; t1++) {
elements_rank_offsets[t1] = elements_rank_offsets[t1 - 1] + elements_rank_sizes[t1 - 1];
}
// find the global/total size
unsigned long long n_elements_global_max = elements_rank_offsets[nranks - 1];
n_elements_global_max += elements_rank_sizes[nranks - 1]; // add the last ranks size to the already accumulated offset values
if (rank == 0) {
OpmLog::debug(fmt::format("In setupDamarisWritingPars(): n_elements_global_max = {}", n_elements_global_max));
}
// Set the paramater so that the Damaris servers can allocate the correct amount of memory for the variabe
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that local ranks are 0 based ?
int temp_int = static_cast<int>(elements_rank_sizes[rank]);
damaris_err = damaris_parameter_set("n_elements_local", &temp_int, sizeof(int));
if (damaris_err != DAMARIS_OK && rank == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_parameter_set(\"n_elements_local\", &temp_int, sizeof(int));");
}
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that n_elements_global_max will fit in a C int type (INT_MAX)
temp_int = static_cast<int>(n_elements_global_max);
damaris_err = damaris_parameter_set("n_elements_total", &temp_int, sizeof(int));
if (damaris_err != DAMARIS_OK && rank == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_parameter_set(\"n_elements_total\", &temp_int, sizeof(int));");
}
// Use damaris_set_position to set the offset in the global size of the array.
// This is used so that output functionality (e.g. HDF5Store) knows global offsets of the data of the ranks
int64_t temp_int64_t[1];
temp_int64_t[0] = static_cast<int64_t>(elements_rank_offsets[rank]);
damaris_err = damaris_set_position("PRESSURE", temp_int64_t);
if (damaris_err != DAMARIS_OK && rank == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_set_position(\"PRESSURE\", temp_int64_t);");
}
damaris_err = damaris_set_position("GLOBAL_CELL_INDEX", temp_int64_t);
if (damaris_err != DAMARIS_OK && rank == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_set_position(\"GLOBAL_CELL_INDEX\", temp_int64_t);");
}
}
} // namespace Opm::DamarisOutput

5
opm/simulators/utils/DamarisOutputModule.hpp
View File

@@ -1,6 +1,7 @@
/*
Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
Copyright 2023 INRIA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
@@ -35,7 +36,5 @@ namespace Opm::DamarisOutput
std::string initDamarisXmlFile();
// Initialize Damaris by filling in th XML file and stroring it in the chosed directory
void initializeDamaris(MPI_Comm comm, int mpiRank, std::string OutputDir, bool enableDamarisOutputCollective);
// Setup Damaris Parameters for writing e.g., grid size and communicator to output "PRESSURE" field
void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid);
} // namespace Opm::DamarisOutput

116
opm/simulators/utils/initDamarisXmlFile.cpp
View File

@@ -1,5 +1,5 @@
/*
Copyright 2022 KerData Research Team, Inria Rennes, Bretagne–Atlantique Research Center
Copyright 2022 2023 Inria, Bretagne–Atlantique Research Center
Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
This file is part of the Open Porous Media project (OPM).
@@ -29,6 +29,10 @@ namespace Opm::DamarisOutput
The entries in the map below will be filled by corresponding Damaris
Keywords.
N.B. Ensure all text items that are to be replaced are quoted with double quotes
e.g. unit="_REPLACE_UNIT_"
*not* unit=_REPLACE_UNIT_
*/
std::string initDamarisXmlFile()
{
@@ -47,11 +51,103 @@ std::string initDamarisXmlFile()
<parameter name="n_elements_local" type="int" value="1" />
<parameter name="n" type="int" value="1" />
<layout name="zonal_layout_usmesh_integer" type="int" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="GLOBAL_CELL_INDEX" layout="zonal_layout_usmesh_integer" type="scalar" visualizable="false" time-varying="false" centering="zonal" />
<layout name="zonal_layout_usmesh" type="double" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="PRESSURE" layout="zonal_layout_usmesh" type="scalar" visualizable="false" select-file="GLOBAL_CELL_INDEX" unit="bar" centering="zonal" store="_MYSTORE_OR_EMPTY_REGEX_" />
<layout name="zonal_layout_usmesh_integer" type="int" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="GLOBAL_CELL_INDEX" layout="zonal_layout_usmesh_integer" type="scalar" visualizable="false" time-varying="false" store="_MYSTORE_OR_EMPTY_REGEX_" centering="zonal" />
<layout name="zonal_layout_usmesh" type="double" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="PRESSURE" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="unstructured_mesh" unit="Bar" centering="zonal" time-varying="true"
select-file="GLOBAL_CELL_INDEX" store="_MYSTORE_OR_EMPTY_REGEX_" script="_PYTHON_XML_NAME_" />
_MORE_VARIABLES_REGEX_
<variable name="MPI_RANK" layout="zonal_layout_usmesh_integer" type="scalar" visualizable="true" mesh="unstructured_mesh" unit="rank" centering="zonal" select-file="GLOBAL_CELL_INDEX" store="#" time-varying="false" script="#" comment="The cells MPI rank"/>
<variable name="KRNSW_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="unstructured_mesh" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="KRNSW_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="PCSWM_GO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="PCSWM_OW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="PPCW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="RS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" comment="Dissolved Gas units Gas Oil Ratio" />
<variable name="RV" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="Bar" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="_PYTHON_XML_NAME_" />
<variable name="SOMAX" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="_MESHNAME_OR_HASH_" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBO" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="1OVERBW" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GASKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GAS_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GAS_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OILKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIL_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIL_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WATKR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WAT_DEN" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WAT_VISC" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="2FBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="4FBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="DFBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GCDI" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GCDM" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="GIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="HTOF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="OIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="RPV" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="S36F" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SEAF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SGAS" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="SWAT" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="TFBF" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WCD" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIP" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIPG" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIPL" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<variable name="WIPR" layout="zonal_layout_usmesh" type="scalar" visualizable="true" mesh="#" unit="" centering="zonal" time-varying="true" select-file="GLOBAL_CELL_INDEX" store="#" script="#" />
<parameter name="n_coords_local" type="int" value="1" />
<layout name="n_coords_layout" type="double" dimensions="n_coords_local" comment="For the individual x, y and z coordinates of the mesh vertices, these values are referenced in the topologies/topo/subelements/connectivity_pg data" />
<group name="coordset/coords/values">
<variable name="x" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="y" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="z" layout="n_coords_layout" type="scalar" visualizable="false" unit="m" script="_PYTHON_XML_NAME_" time-varying="false" />
</group>
<parameter name="n_connectivity_ph" type="int" value="1" />
<layout name="n_connections_layout_ph" type="int" dimensions="n_connectivity_ph" comment="Layout for connectivities " />
<parameter name="n_offsets_types_ph" type="int" value="1" />
<layout name="n_offsets_layout_ph" type="int" dimensions="n_offsets_types_ph+1" comment="Layout for the offsets_ph" />
<layout name="n_types_layout_ph" type="char" dimensions="n_offsets_types_ph" comment="Layout for the types_ph " />
<group name="topologies/topo/elements">
<variable name="connectivity" layout="n_connections_layout_ph" type="scalar" visualizable="false" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="offsets" layout="n_offsets_layout_ph" type="scalar" visualizable="false" script="_PYTHON_XML_NAME_" time-varying="false" />
<variable name="types" layout="n_types_layout_ph" type="scalar" visualizable="false" script="_PYTHON_XML_NAME_" time-varying="false" />
</group>
<mesh name="unstructured_mesh" type="unstructured" topology="3" time-varying="false"
comment="This Mesh definition is for connection with Paraview.
This definition references the variables that define an unstructured mesh specified above." >
<coord name="coordset/coords/values/x" unit="m" />
<coord name="coordset/coords/values/y" unit="m" />
<coord name="coordset/coords/values/z" unit="m" />
<vertex_global_id name="#" offset="0" />
<element_offsets name="topologies/topo/elements/offsets" />
<section_types name="topologies/topo/elements/types" />
<section_sizes name="#" />
<section_connectivity name="topologies/topo/elements/connectivity" />
<polyhedral_cell_faces_connectivity name="#" />
<polyhedral_cell_faces_offsets name="#" />
<polyhedral_n_faces_per_cell name="#" />
</mesh>
</data>
<storage>
@@ -62,10 +158,18 @@ std::string initDamarisXmlFile()
</store>
</storage>
<scripts>
<pyscript name="PythonScript" file="_PYTHON_OR_EMPTY_REGEX_" language="python" frequency="1" scheduler-file="" nthreads="0" keep-workers="no" />
</scripts>
<!-- paraview update-frequency="1" write-vtk="0" write-vtk-binary="false">
<script>_PARAVIEWPY_OR_EMPTY_REGEX_</script>
</paraview -->
<actions>
</actions>
<log FileName="_PATH_REGEX_/damaris_log/exa_dbg" RotationSize="5" LogFormat="[%TimeStamp%]: %Message%" Flush="True" LogLevel="debug" />
<log FileName="_PATH_REGEX_/damaris_log/opm-flow" RotationSize="5" LogFormat="[%TimeStamp%]: %Message%" Flush="true" LogLevel="info" />
</simulation>)V0G0N";