OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4121 from ElyesAhmed/damaris_integ_v3

Browse Source 
Damaris integ v3
This commit is contained in:
Atgeirr Flø Rasmussen 2022-10-13 14:02:30 +02:00 committed by GitHub
commit 2064e8725f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 534 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
CMakeLists.txt
View File

@ -31,6 +31,7 @@ option(USE_CHOW_PATEL_ILU "Use the iterative ILU by Chow and Patel?" OFF)
option(USE_CHOW_PATEL_ILU_GPU "Run iterative ILU decomposition on GPU? Requires USE_CHOW_PATEL_ILU" OFF)
option(USE_CHOW_PATEL_ILU_GPU_PARALLEL "Try to use more parallelism on the GPU during the iterative ILU decomposition? Requires USE_CHOW_PATEL_ILU_GPU" OFF)
option(BUILD_FLOW_ALU_GRID "Build flow blackoil with alu grid" OFF)
option(USE_DAMARIS_LIB "Use the Damaris library for asynchronous I/O?" OFF)
# The following was copied from CMakeLists.txt in opm-common.
# TODO: factor out the common parts in opm-common and opm-simulator as a cmake module
@ -63,7 +64,6 @@ if (OPM_ENABLE_PYTHON)
set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE})
endif()
if(SIBLING_SEARCH AND NOT opm-common_DIR)
# guess the sibling dir
get_filename_component(_leaf_dir_name ${PROJECT_BINARY_DIR} NAME)
@ -297,7 +297,6 @@ if(OpenCL_FOUND)
endif()
endif()
# read the list of components from this file (in the project directory);
# it should set various lists with the names of the files to include
include (CMakeLists_files.cmake)
@ -315,6 +314,8 @@ macro (config_hook)
list(APPEND EXTRA_INCLUDES SYSTEM ${PROJECT_SOURCE_DIR}/external/fmtlib/include)
endif()
include_directories(${EXTRA_INCLUDES})
include(UseDamaris)
endmacro (config_hook)
macro (prereqs_hook)
@ -335,7 +336,8 @@ endmacro (files_hook)
macro (tests_hook)
endmacro (tests_hook)
# all setup common to the OPM library modules is done here
include (OpmLibMain)
@ -544,6 +546,11 @@ if(HAVE_FPGA)
ExternalProject_Get_Property(FPGA_library binary_dir)
target_link_libraries(opmsimulators PUBLIC ${binary_dir}/fpga_lib_alveo_u280.a)
endif()
if(Damaris_FOUND)
target_link_libraries(opmsimulators PUBLIC damaris)
endif()
install(DIRECTORY doc/man1 DESTINATION ${CMAKE_INSTALL_MANDIR}
FILES_MATCHING PATTERN "*.1")

5
CMakeLists_files.cmake
View File

@ -108,6 +108,11 @@ list (APPEND MAIN_SOURCE_FILES
opm/simulators/wells/WGState.cpp
)
if (DAMARIS_FOUND)
list (APPEND MAIN_SOURCE_FILES opm/simulators/utils/DamarisOutputModule.cpp)
list (APPEND MAIN_SOURCE_FILES opm/simulators/utils/DamarisKeywords.cpp)
list (APPEND MAIN_SOURCE_FILES opm/simulators/utils/initDamarisXmlFile.cpp)
endif()
if(CUDA_FOUND)
list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/cuda/cusparseSolverBackend.cu)
list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/bda/cuda/cuWellContributions.cu)

8
ebos/eclgenericoutputblackoilmodule.hh
View File

@ -47,6 +47,14 @@ class EclipseState;
template<class FluidSystem, class Scalar>
class EclGenericOutputBlackoilModule {
public:
Scalar* getPRESSURE_ptr(void) {
return (this->oilPressure_.data()) ;
};
int getPRESSURE_size( void ) {
return (this->oilPressure_.size()) ;
};
// write cumulative production and injection reports to output
void outputCumLog(size_t reportStepNum,
const bool substep,

17
ebos/eclproblem.hh
View File

@ -431,12 +431,25 @@ template<class TypeTag>
struct EnableEclOutput<TypeTag,TTag::EclBaseProblem> {
static constexpr bool value = true;
};
#ifdef HAVE_DAMARIS
//! Enable the Damaris output by default
template<class TypeTag>
struct EnableDamarisOutput<TypeTag, TTag::EclBaseProblem> {
static constexpr bool value = false;
};
// If Damaris is available, write specific variable output in parallel
template<class TypeTag>
struct EnableDamarisOutputCollective<TypeTag, TTag::EclBaseProblem> {
static constexpr bool value = true;
};
#endif
// If available, write the ECL output in a non-blocking manner
template<class TypeTag>
struct EnableAsyncEclOutput<TypeTag, TTag::EclBaseProblem> {
static constexpr bool value = true;
};
// Write ESMRY file for fast loading of summary data
template<class TypeTag>
struct EnableEsmry<TypeTag, TTag::EclBaseProblem> {
@ -695,6 +708,10 @@ public:
EWOMS_REGISTER_PARAM(TypeTag, bool, EnableEclOutput,
"Write binary output which is compatible with the commercial "
"Eclipse simulator");
#ifdef HAVE_DAMARIS
EWOMS_REGISTER_PARAM(TypeTag, bool, EnableDamarisOutput,
"Write a specific variable using Damaris in a separate core");
#endif
EWOMS_REGISTER_PARAM(TypeTag, bool, EclOutputDoublePrecision,
"Tell the output writer to use double precision. Useful for 'perfect' restarts");
EWOMS_REGISTER_PARAM(TypeTag, unsigned, RestartWritingInterval,

55
ebos/eclwriter.hh
View File

@ -43,6 +43,11 @@
#include <stdexcept>
#include <string>
#if HAVE_DAMARIS
#include <opm/simulators/utils/DamarisOutputModule.hpp>
#endif
namespace Opm::Properties {
template<class TypeTag, class MyTypeTag>
@ -57,11 +62,20 @@ template<class TypeTag, class MyTypeTag>
struct EclOutputDoublePrecision {
using type = UndefinedProperty;
};
#ifdef HAVE_DAMARIS
template<class TypeTag, class MyTypeTag>
struct EnableDamarisOutput {
using type = UndefinedProperty;
};
template<class TypeTag, class MyTypeTag>
struct EnableDamarisOutputCollective {
using type = UndefinedProperty;
};
#endif
template<class TypeTag, class MyTypeTag>
struct EnableEsmry {
using type = UndefinedProperty;
};
} // namespace Opm::Properties
namespace Opm {
@ -116,6 +130,10 @@ public:
EWOMS_REGISTER_PARAM(TypeTag, bool, EnableAsyncEclOutput,
"Write the ECL-formated results in a non-blocking way (i.e., using a separate thread).");
#ifdef HAVE_DAMARIS
EWOMS_REGISTER_PARAM(TypeTag, bool, EnableDamarisOutputCollective,
"Write output via Damaris using parallel HDF5 to get single file per timestep instead of one per Damaris core.");
#endif
EWOMS_REGISTER_PARAM(TypeTag, bool, EnableEsmry,
"Write ESMRY file for fast loading of summary data.");
}
@ -135,6 +153,9 @@ public:
EWOMS_GET_PARAM(TypeTag, bool, EnableAsyncEclOutput), EWOMS_GET_PARAM(TypeTag, bool, EnableEsmry))
, simulator_(simulator)
{
#ifdef HAVE_DAMARIS
this->damarisUpdate_ = enableDamarisOutput_();
#endif
this->eclOutputModule_ = std::make_unique<EclOutputBlackOilModule<TypeTag>>(simulator, this->wbp_index_list_, this->collectToIORank_);
this->wbp_index_list_.clear();
}
@ -281,10 +302,30 @@ public:
void writeOutput(bool isSubStep)
{
const int reportStepNum = simulator_.episodeIndex() + 1;
this->prepareLocalCellData(isSubStep, reportStepNum);
this->eclOutputModule_->outputErrorLog(simulator_.gridView().comm());
#ifdef HAVE_DAMARIS
if (EWOMS_GET_PARAM(TypeTag, bool, EnableDamarisOutput)) {
// N.B. damarisUpdate_ should be set to true if at any time the model geometry changes
if (this->damarisUpdate_) {
const auto& gridView = simulator_.gridView();
const int numElements = gridView.size(/*codim=*/0); // This it is the local ranks model size.
Opm::DamarisOutput::setupDamarisWritingPars(simulator_.vanguard().grid().comm(), numElements);
// By default we assume static grid
this->damarisUpdate_ = false;
}
if (!isSubStep) {
data::Solution localCellData = {};
this->eclOutputModule_->assignToSolution(localCellData);
// Output the PRESSURE field
if (this->eclOutputModule_->getPRESSURE_ptr() != nullptr) {
damaris_write("PRESSURE", (void*)this->eclOutputModule_->getPRESSURE_ptr());
damaris_end_iteration();
}
}
}
#endif
// output using eclWriter if enabled
auto localWellData = simulator_.problem().wellModel().wellData();
auto localGroupAndNetworkData = simulator_.problem().wellModel()
@ -300,7 +341,7 @@ public:
// add cell data to perforations for Rft output
this->eclOutputModule_->addRftDataToWells(localWellData, reportStepNum);
}
if (this->collectToIORank_.isParallel()|| this->collectToIORank_.doesNeedReordering()) {
this->collectToIORank_.collect(localCellData,
eclOutputModule_->getBlockData(),
@ -422,7 +463,10 @@ public:
private:
static bool enableEclOutput_()
{ return EWOMS_GET_PARAM(TypeTag, bool, EnableEclOutput); }
#ifdef HAVE_DAMARIS
static bool enableDamarisOutput_()
{ return EWOMS_GET_PARAM(TypeTag, bool, EnableDamarisOutput); }
#endif
const EclipseState& eclState() const
{ return simulator_.vanguard().eclState(); }
@ -503,6 +547,9 @@ private:
Simulator& simulator_;
std::unique_ptr<EclOutputBlackOilModule<TypeTag>> eclOutputModule_;
Scalar restartTimeStepSize_;
#ifdef HAVE_DAMARIS
bool damarisUpdate_ = false; ///< Whenever this is true writeOutput() will set up Damaris offsets of model fields
#endif
};
} // namespace Opm

2
opm-simulators-prereqs.cmake
View File

@ -18,6 +18,7 @@ set (opm-simulators_CONFIG_VAR
DUNE_ISTL_VERSION_MINOR
DUNE_ISTL_VERSION_REVISION
HAVE_SUITESPARSE_UMFPACK
HAVE_DAMARIS
)
# dependencies
@ -44,6 +45,7 @@ set (opm-simulators_DEPS
"opm-material REQUIRED"
"opm-grid REQUIRED"
"opm-models REQUIRED"
"Damaris 1.7"
)
find_package_deps(opm-simulators)

71
opm/simulators/flow/Main.hpp
View File

@ -74,6 +74,10 @@
#include <opm/simulators/utils/ParallelEclipseState.hpp>
#endif
#if HAVE_DAMARIS
#include <opm/simulators/utils/DamarisOutputModule.hpp>
#endif
#include <cassert>
#include <cstdlib>
#include <filesystem>
@ -166,6 +170,22 @@ public:
EclGenericVanguard::setCommunication(nullptr);
#if HAVE_DAMARIS
if (enableDamarisOutput_) {
int err;
if (isSimulationRank_) {
err = damaris_stop();
if (err != DAMARIS_OK) {
std::cerr << "ERROR: Damaris library produced an error result for damaris_stop()" << std::endl;
}
}
err = damaris_finalize();
if (err != DAMARIS_OK) {
std::cerr << "ERROR: Damaris library produced an error result for damaris_finalize()" << std::endl;
}
}
#endif // HAVE_DAMARIS
#if HAVE_MPI && !HAVE_DUNE_FEM
MPI_Finalize();
#endif
@ -377,11 +397,6 @@ private:
externalSetupTimer.start();
handleVersionCmdLine_(argc_, argv_);
#if HAVE_DUNE_FEM
int mpiRank = Dune::Fem::MPIManager::rank();
#else
int mpiRank = EclGenericVanguard::comm().rank();
#endif
// we always want to use the default locale, and thus spare us the trouble
// with incorrect locale settings.
@ -410,11 +425,6 @@ private:
return false; // Whether to run the simulator
}
FileOutputMode outputMode = FileOutputMode::OUTPUT_NONE;
outputCout_ = false;
if (mpiRank == 0)
outputCout_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableTerminalOutput);
std::string deckFilename;
std::string outputDir;
if ( eclipseState_ ) {
@ -423,8 +433,43 @@ private:
}
else {
deckFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, EclDeckFileName);
outputDir = EWOMS_GET_PARAM(PreTypeTag, std::string, OutputDir);
}
#if HAVE_DAMARIS
enableDamarisOutput_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutput);
if (enableDamarisOutput_) {
if (!outputDir.empty()) {
ensureOutputDirExists(outputDir);
}
// By default EnableDamarisOutputCollective is true so all simulation results will
// be written into one single file for each iteration using Parallel HDF5.
// It set to false, FilePerCore mode is used in Damaris, then simulation results in each
// node are aggregated by dedicated Damaris cores and stored to separate files per Damaris core.
// Irrespective of mode, output is written asynchronously at the end of each timestep.
const bool enableDamarisOutputCollective = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutputCollective);
// Using the ModifyModel class to set the XML file for Damaris.
DamarisOutput::initializeDamaris(EclGenericVanguard::comm(), EclGenericVanguard::comm().rank(), outputDir, enableDamarisOutputCollective);
int is_client;
MPI_Comm new_comm;
int err = damaris_start(&is_client);
isSimulationRank_ = (is_client > 0);
if (isSimulationRank_ && err == DAMARIS_OK) {
damaris_client_comm_get(&new_comm);
EclGenericVanguard::setCommunication(std::make_unique<Parallel::Communication>(new_comm));
} else {
return false;
}
}
#endif // HAVE_DAMARIS
int mpiRank = EclGenericVanguard::comm().rank();
FileOutputMode outputMode = FileOutputMode::OUTPUT_NONE;
outputCout_ = false;
if (mpiRank == 0)
outputCout_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableTerminalOutput);
if (deckFilename.empty()) {
if (mpiRank == 0) {
std::cerr << "No input case given. Try '--help' for a usage description.\n";
@ -454,9 +499,6 @@ private:
try {
auto python = std::make_shared<Python>();
const bool init_from_restart_file = !EWOMS_GET_PARAM(PreTypeTag, bool, SchedRestart);
if (outputDir.empty())
outputDir = EWOMS_GET_PARAM(PreTypeTag, std::string, OutputDir);
const bool allRanksDbgPrtLog = EWOMS_GET_PARAM(PreTypeTag, bool,
EnableLoggingFalloutWarning);
outputMode = setupLogging(mpiRank,
@ -787,6 +829,9 @@ private:
// To demonstrate run with non_world_comm
bool test_split_comm_ = false;
bool isSimulationRank_ = true;
#if HAVE_DAMARIS
bool enableDamarisOutput_ = false;
#endif
};
} // namespace Opm

63
opm/simulators/utils/DamarisKeywords.cpp Normal file
View File

@ -0,0 +1,63 @@
/*
Copyright 2021 Equinor.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <string>
#include <map>
/*
Below is the Damaris Keywords supported by Damaris to be filled
in the built-in XML file.
The entries in the map below will be filled by the corresponding
Damaris Keywords. Yet, only output directory and FileMode are to
be chosen by the user
*/
namespace Opm::DamarisOutput
{
std::map<std::string, std::string>
DamarisKeywords(std::string OutputDir, bool enableDamarisOutputCollective)
{
if (enableDamarisOutputCollective) {
std::map<std::string, std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_", "536870912"},
{"_DC_REGEX_", "1"},
{"_DN_REGEX_", "0"},
{"_File_Mode", "Collective"},
{"_MORE_VARIABLES_REGEX_", ""},
{"_PATH_REGEX_", OutputDir},
{"_MYSTORE_OR_EMPTY_REGEX_", "MyStore"},
};
return damaris_keywords;
} else {
std::map<std::string, std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_", "536870912"},
{"_DC_REGEX_", "1"},
{"_DN_REGEX_", "0"},
{"_File_Mode", "FilePerCore"},
{"_MORE_VARIABLES_REGEX_", ""},
{"_PATH_REGEX_", OutputDir},
{"_MYSTORE_OR_EMPTY_REGEX_", "MyStore"},
};
return damaris_keywords;
}
}
} // namespace Opm::DamarisOutput

42
opm/simulators/utils/DamarisKeywords.hpp Normal file
View File

@ -0,0 +1,42 @@
/*
Copyright 2021 Equinor.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPM_DAMARISKEYWORDS_HEADER_INCLUDED
#define OPM_DAMARISKEYWORDS_HEADER_INCLUDED
#include <string>
#include <map>
/*
Below is the std::map with the keywords that are supported by Damaris.
Most entries in the map below are not critical ('static') and will not
be changed. We only allow changing FileMode together with output directory
*/
namespace Opm::DamarisOutput
{
std::map<std::string,std::string> DamarisKeywords(std::string outputDir, bool enableDamarisOutputCollective);
} // namespace Opm::DamarisOutput
#endif

141
opm/simulators/utils/DamarisOutputModule.cpp Normal file
View File

@ -0,0 +1,141 @@
/*
Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#define XSD_CXX11_TEMPLATE_ALIAS 1
#include <damaris/model/ModifyModel.hpp>
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <opm/simulators/utils/DamarisOutputModule.hpp>
#include <opm/simulators/utils/ParallelCommunication.hpp>
#include <opm/common/OpmLog/OpmLog.hpp>
#include <fmt/format.h>
namespace Opm::DamarisOutput
{
std::string initDamarisXmlFile(); // Defined in initDamarisXMLFile.cpp, to avoid messing up this file.
void
initializeDamaris(MPI_Comm comm, int mpiRank, std::string outputDir, bool enableDamarisOutputCollective)
{
if (outputDir.empty()) {
outputDir = ".";
}
// Prepare the XML file
std::string damaris_config_xml = initDamarisXmlFile();
damaris::model::ModifyModel myMod = damaris::model::ModifyModel(damaris_config_xml);
// The map will make it precise the output directory and FileMode (either FilePerCore or Collective storage)
// The map file find all occurences of the string in position 1 and repalce it/them with string in position 2
std::map<std::string, std::string> find_replace_map = DamarisKeywords(outputDir, enableDamarisOutputCollective);
myMod.RepalceWithRegEx(find_replace_map);
std::string damaris_xml_filename_str = outputDir + "/damaris_config.xml";
if (mpiRank == 0) {
myMod.SaveXMLStringToFile(damaris_xml_filename_str);
}
int damaris_err;
/* Get the name of the Damaris input file from an environment variable if available */
const char* cs_damaris_xml_file = getenv("FLOW_DAMARIS_XML_FILE");
if (cs_damaris_xml_file != NULL) {
std::cout << "INFO: initializing Damaris from environment variable FLOW_DAMARIS_XML_FILE: "
<< cs_damaris_xml_file << std::endl;
damaris_err = damaris_initialize(cs_damaris_xml_file, MPI_COMM_WORLD);
if (damaris_err != DAMARIS_OK) {
std::cerr << "ERROR: damaris_initialize() error via FLOW_DAMARIS_XML_FILE=" << cs_damaris_xml_file
<< std::endl;
}
} else {
std::cout << "INFO: initializing Damaris using internally built file:" << damaris_xml_filename_str << std::endl;
damaris_err = damaris_initialize(damaris_xml_filename_str.c_str(), comm);
if (damaris_err != DAMARIS_OK) {
std::cerr << "ERROR: damaris_initialize() error via built file:" << std::endl << myMod.GetConfigString();
}
}
}
void
setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid)
{
int damaris_err = DAMARIS_OK;
const int nranks = comm.size();
const int rank = comm.rank();
std::vector<unsigned long long> elements_rank_sizes(nranks); // one for each rank -- to be gathered from each client rank
std::vector<unsigned long long> elements_rank_offsets(nranks); // one for each rank, first one 0 -- to be computed - Probably could use MPI_Scan()!
// n_elements_local_grid should be the full model size
const unsigned long long n_elements_local = n_elements_local_grid;
// Left in for debugging, but commented out to avoid spamming the terminal from non-output ranks.
// std::cout << "INFO (" << rank << "): n_elements_local_grid = " << n_elements_local_grid << std::endl;
// This gets the n_elements_local from all ranks and copies them to a std::vector of all the values on all ranks
// (elements_rank_sizes[]).
comm.allgather(&n_elements_local, 1, elements_rank_sizes.data());
elements_rank_offsets[0] = 0ULL;
// This scan makes the offsets to the start of each ranks grid section if each local grid data was concatenated (in
// rank order)
for (int t1 = 1; t1 < nranks; t1++) {
elements_rank_offsets[t1] = elements_rank_offsets[t1 - 1] + elements_rank_sizes[t1 - 1];
}
// find the global/total size
unsigned long long n_elements_global_max = elements_rank_offsets[nranks - 1];
n_elements_global_max += elements_rank_sizes[nranks - 1]; // add the last ranks size to the already accumulated offset values
if (rank == 0) {
OpmLog::debug(fmt::format("In setupDamarisWritingPars(): n_elements_global_max = {}", n_elements_global_max));
}
// Set the paramater so that the Damaris servers can allocate the correct amount of memory for the variabe
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that local ranks are 0 based ?
int temp_int = static_cast<int>(elements_rank_sizes[rank]);
damaris_err = damaris_parameter_set("n_elements_local", &temp_int, sizeof(int));
if (damaris_err != DAMARIS_OK && rank == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_parameter_set(\"n_elements_local\", &temp_int, sizeof(int));");
}
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that n_elements_global_max will fit in a C int type (INT_MAX)
temp_int = static_cast<int>(n_elements_global_max);
damaris_err = damaris_parameter_set("n_elements_total", &temp_int, sizeof(int));
if (damaris_err != DAMARIS_OK && rank == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_parameter_set(\"n_elements_total\", &temp_int, sizeof(int));");
}
// Use damaris_set_position to set the offset in the global size of the array.
// This is used so that output functionality (e.g. HDF5Store) knows global offsets of the data of the ranks
int64_t temp_int64_t[1];
temp_int64_t[0] = static_cast<int64_t>(elements_rank_offsets[rank]);
damaris_err = damaris_set_position("PRESSURE", temp_int64_t);
if (damaris_err != DAMARIS_OK && rank == 0) {
OpmLog::error("Damaris library produced an error result for "
"damaris_set_position(\"PRESSURE\", temp_int64_t);");
}
}
} // namespace Opm::DamarisOutput

41
opm/simulators/utils/DamarisOutputModule.hpp Normal file
View File

@ -0,0 +1,41 @@
/*
Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <string>
#include <Damaris.h>
#include <opm/simulators/utils/ParallelCommunication.hpp>
/*
Below is the XML file for Damaris that is supported by Damaris.
The entries in the map below will be filled by corresponding Damaris
Keywords.
*/
namespace Opm::DamarisOutput
{
// Initialize an XML file
std::string initDamarisXmlFile();
// Initialize Damaris by filling in th XML file and stroring it in the chosed directory
void initializeDamaris(MPI_Comm comm, int mpiRank, std::string OutputDir, bool enableDamarisOutputCollective);
// Setup Damaris Parameters for writing e.g., grid size and communicator to output "PRESSURE" field
void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid);
} // namespace Opm::DamarisOutput

73
opm/simulators/utils/initDamarisXmlFile.cpp Normal file
View File

@ -0,0 +1,73 @@
/*
Copyright 2022 KerData Research Team, Inria Rennes, Bretagne–Atlantique Research Center
Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <string>
namespace Opm::DamarisOutput
{
/*
Below is the XML file for Damaris that is supported by Damaris.
The entries in the map below will be filled by corresponding Damaris
Keywords.
*/
std::string initDamarisXmlFile()
{
std::string init_damaris = R"V0G0N(<?xml version="1.0"?>
<simulation name="opm-flow" language="c" xmlns="http://damaris.gforge.inria.fr/damaris/model">
<architecture>
<domains count="1"/>
<dedicated cores="_DC_REGEX_" nodes="_DN_REGEX_"/>
<buffer name="buffer" size="_SHMEM_BUFFER_BYTES_REGEX_" />
<placement />
<queue name="queue" size="300" />
</architecture>
<data>
<parameter name="n_elements_total" type="int" value="1" />
<parameter name="n_elements_local" type="int" value="1" />
<parameter name="n" type="int" value="1" />
<layout name="zonal_layout_usmesh" type="double" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="PRESSURE" layout="zonal_layout_usmesh" type="scalar" visualizable="false" unit="Pa" centering="zonal" store="_MYSTORE_OR_EMPTY_REGEX_" />
_MORE_VARIABLES_REGEX_
</data>
<storage>
<store name="MyStore" type="HDF5">
<option key="FileMode">_File_Mode</option>
<option key="XDMFMode">NoIteration</option>
<option key="FilesPath">_PATH_REGEX_/</option>
</store>
</storage>
<actions>
</actions>
<log FileName="_PATH_REGEX_/damaris_log/exa_dbg" RotationSize="5" LogFormat="[%TimeStamp%]: %Message%" Flush="True" LogLevel="debug" />
</simulation>)V0G0N";
return init_damaris;
}
} // namespace Opm::DamarisOutput

37
opm/simulators/utils/readDeck.cpp
View File

@ -93,23 +93,6 @@ namespace {
->setMessageLimiter(std::make_shared<Opm::MessageLimiter>(10, limits));
}
void ensureOutputDirExists_(const std::string& cmdline_output_dir)
{
namespace fs = std::filesystem;
if (! fs::is_directory(cmdline_output_dir)) {
try {
fs::create_directories(cmdline_output_dir);
}
catch (...) {
throw std::runtime_error {
fmt::format("Creation of output directory '{}' failed",
cmdline_output_dir)
};
}
}
}
void loadObjectsFromRestart(const Opm::Deck& deck,
const Opm::Parser& parser,
const Opm::ParseContext& parseContext,
@ -376,6 +359,24 @@ namespace {
// ---------------------------------------------------------------------------
void Opm::ensureOutputDirExists(const std::string& cmdline_output_dir)
{
namespace fs = std::filesystem;
if (! fs::is_directory(cmdline_output_dir)) {
try {
fs::create_directories(cmdline_output_dir);
}
catch (...) {
throw std::runtime_error {
fmt::format("Creation of output directory '{}' failed",
cmdline_output_dir)
};
}
}
}
// Setup the OpmLog backends
Opm::FileOutputMode
Opm::setupLogging(const int mpi_rank_,
@ -387,7 +388,7 @@ Opm::setupLogging(const int mpi_rank_,
const bool allRanksDbgLog)
{
if (!cmdline_output_dir.empty()) {
ensureOutputDirExists_(cmdline_output_dir);
ensureOutputDirExists(cmdline_output_dir);
}
// create logFile

4
opm/simulators/utils/readDeck.hpp
View File

@ -57,6 +57,10 @@ enum class FileOutputMode {
OUTPUT_ALL = 3,
};
// Ensure that a directory exists, creating it if it does not.
void
ensureOutputDirExists(const std::string& cmdline_output_dir);
// Setup the OpmLog backends
FileOutputMode
setupLogging(int mpi_rank_,