From 8c3400f562bb35852ca72fb3db4df5db542cbe97 Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Thu, 9 Feb 2023 23:30:02 +0100 Subject: [PATCH] added: parallel support to HDF5File / HDF5Serializer --- CMakeLists.txt | 28 +++ ebos/hdf5serializer.hh | 19 +- examples/opmrst_inspect.cpp | 10 +- .../SimulatorFullyImplicitBlackoilEbos.hpp | 37 +++- opm/simulators/utils/HDF5File.cpp | 171 ++++++++++++++---- opm/simulators/utils/HDF5File.hpp | 36 +++- tests/test_HDF5File.cpp | 53 +++++- tests/test_HDF5File_Parallel.cpp | 79 ++++++++ tests/test_HDF5Serializer.cpp | 33 +++- tests/test_HDF5Serializer_Parallel.cpp | 120 ++++++++++++ 10 files changed, 518 insertions(+), 68 deletions(-) create mode 100644 tests/test_HDF5File_Parallel.cpp create mode 100644 tests/test_HDF5Serializer_Parallel.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bef3bc997..5ae79099a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,6 +91,10 @@ endif() find_package(opm-common REQUIRED) +if(USE_MPI) + set(HDF5_PREFER_PARALLEL TRUE) +endif() + include(OpmInit) OpmSetPolicies() @@ -340,6 +344,30 @@ opm_add_test(test_broadcast -b ${PROJECT_BINARY_DIR} ) +opm_add_test(test_HDF5File_Parallel + DEPENDS "opmsimulators" + LIBRARIES opmsimulators ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} + SOURCES + tests/test_HDF5File_Parallel.cpp + CONDITION + HDF5_FOUND AND MPI_FOUND AND Boost_UNIT_TEST_FRAMEWORK_FOUND + DRIVER_ARGS + -n 4 + -b ${PROJECT_BINARY_DIR} +) + +opm_add_test(test_HDF5Serializer_Parallel + DEPENDS "opmsimulators" + LIBRARIES opmsimulators ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} + SOURCES + tests/test_HDF5Serializer_Parallel.cpp + CONDITION + HDF5_FOUND AND MPI_FOUND AND Boost_UNIT_TEST_FRAMEWORK_FOUND + DRIVER_ARGS + -n 4 + -b ${PROJECT_BINARY_DIR} +) + include(OpmBashCompletion) if (NOT BUILD_FLOW) diff --git a/ebos/hdf5serializer.hh b/ebos/hdf5serializer.hh index c00975b62..bc669a394 100644 --- a/ebos/hdf5serializer.hh +++ b/ebos/hdf5serializer.hh @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -36,9 +37,11 @@ namespace Opm { //! \brief Class for (de-)serializing using HDF5. class HDF5Serializer : public Serializer { public: - HDF5Serializer(const std::string& fileName, HDF5File::OpenMode mode) + HDF5Serializer(const std::string& fileName, + HDF5File::OpenMode mode, + Parallel::Communication comm) : Serializer(m_packer_priv) - , m_h5file(fileName, mode) + , m_h5file(fileName, mode, comm) {} //! \brief Serialize and write data to restart file. @@ -47,7 +50,8 @@ public: template void write(T& data, const std::string& group, - const std::string& dset) + const std::string& dset, + HDF5File::DataSetMode mode = HDF5File::DataSetMode::PROCESS_SPLIT) { try { this->pack(data); @@ -56,7 +60,7 @@ public: throw; } - m_h5file.write(group, dset, m_buffer); + m_h5file.write(group, dset, m_buffer, mode); } //! \brief Writes a header to the file. @@ -80,7 +84,7 @@ public: m_packSize = std::numeric_limits::max(); throw; } - m_h5file.write("/", "simulator_info", m_buffer); + m_h5file.write("/", "simulator_info", m_buffer, HDF5File::DataSetMode::ROOT_ONLY); } //! \brief Read data and deserialize from restart file. @@ -89,9 +93,10 @@ public: template void read(T& data, const std::string& group, - const std::string& dset) + const std::string& dset, + HDF5File::DataSetMode mode = HDF5File::DataSetMode::PROCESS_SPLIT) { - m_h5file.read(group, dset, m_buffer); + m_h5file.read(group, dset, m_buffer, mode); this->unpack(data); } diff --git a/examples/opmrst_inspect.cpp b/examples/opmrst_inspect.cpp index 2f310f6d2..a2bd6ac18 100644 --- a/examples/opmrst_inspect.cpp +++ b/examples/opmrst_inspect.cpp @@ -22,6 +22,7 @@ #include #include +#include #include @@ -38,7 +39,14 @@ int main(int argc, char** argv) return 1; } - Opm::HDF5Serializer ser(argv[1], Opm::HDF5File::OpenMode::READ); +#if HAVE_MPI + Opm::Parallel::Communication comm(MPI_COMM_SELF); +#else + Opm::Parallel::Communication comm(); +#endif + + Dune::MPIHelper::instance(argc, argv); + Opm::HDF5Serializer ser(argv[1], Opm::HDF5File::OpenMode::READ, comm); std::tuple,int> header; try { diff --git a/opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp b/opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp index fc995bda0..386874d8a 100644 --- a/opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp +++ b/opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp @@ -569,8 +569,9 @@ protected: return; } - int nextStep = timer.currentStepNum(); + OPM_BEGIN_PARALLEL_TRY_CATCH(); + int nextStep = timer.currentStepNum(); if ((saveStep_ != -1 && nextStep == saveStep_) || (saveStride_ != -1 && (nextStep % saveStride_) == 0)) { #if !HAVE_HDF5 @@ -580,7 +581,9 @@ protected: if (nextStep == saveStride_ || nextStep == saveStep_) { std::filesystem::remove(saveFile_); } - HDF5Serializer writer(saveFile_, HDF5File::OpenMode::APPEND); + HDF5Serializer writer(saveFile_, + HDF5File::OpenMode::APPEND, + EclGenericVanguard::comm()); if (nextStep == saveStride_ || nextStep == saveStep_) { std::ostringstream str; Parameters::printValues(str); @@ -592,10 +595,14 @@ protected: EclGenericVanguard::comm().size()); } writer.write(*this, groupName, "simulator_data"); - writer.write(timer, groupName, "simulator_timer"); + writer.write(timer, groupName, "simulator_timer", + HDF5File::DataSetMode::ROOT_ONLY); OpmLog::info("Serialized state written for report step " + std::to_string(nextStep)); #endif } + + OPM_END_PARALLEL_TRY_CATCH("Error saving serialized state: ", + EclGenericVanguard::comm()); } //! \brief Load timer info from serialized state. @@ -605,13 +612,22 @@ protected: OpmLog::error("Loading of serialized state requested, but no HDF5 support available."); loadStep_ = -1; #else - HDF5Serializer reader(saveFile_, HDF5File::OpenMode::READ); - if (loadStep_ == 0) + OPM_BEGIN_PARALLEL_TRY_CATCH(); + + HDF5Serializer reader(saveFile_, + HDF5File::OpenMode::READ, + EclGenericVanguard::comm()); + + if (loadStep_ == 0) { loadStep_ = reader.lastReportStep(); + } OpmLog::info("Loading serialized state for report step " + std::to_string(loadStep_)); const std::string groupName = "/report_step/" + std::to_string(loadStep_); - reader.read(timer, groupName, "simulator_timer"); + reader.read(timer, groupName, "simulator_timer", HDF5File::DataSetMode::ROOT_ONLY); + + OPM_END_PARALLEL_TRY_CATCH("Error loading serialized state: ", + EclGenericVanguard::comm()); #endif } @@ -619,9 +635,16 @@ protected: void loadSimulatorState() { #if HAVE_HDF5 - HDF5Serializer reader(saveFile_, HDF5File::OpenMode::READ); + OPM_BEGIN_PARALLEL_TRY_CATCH(); + + HDF5Serializer reader(saveFile_, + HDF5File::OpenMode::READ, + EclGenericVanguard::comm()); const std::string groupName = "/report_step/" + std::to_string(loadStep_); reader.read(*this, groupName, "simulator_data"); + + OPM_END_PARALLEL_TRY_CATCH("Error loading serialized state: ", + EclGenericVanguard::comm()); #endif } diff --git a/opm/simulators/utils/HDF5File.cpp b/opm/simulators/utils/HDF5File.cpp index e4237b48a..4818c63a8 100644 --- a/opm/simulators/utils/HDF5File.cpp +++ b/opm/simulators/utils/HDF5File.cpp @@ -24,6 +24,8 @@ #include +#include + #include #include @@ -46,18 +48,32 @@ bool groupExists(hid_t parent, const std::string& path) namespace Opm { -HDF5File::HDF5File(const std::string& fileName, OpenMode mode) +HDF5File::HDF5File(const std::string& fileName, + OpenMode mode, + Parallel::Communication comm) + : comm_(comm) { bool exists = std::filesystem::exists(fileName); + hid_t acc_tpl = H5P_DEFAULT; + if (comm.size() > 1) { +#if HAVE_MPI + MPI_Info info = MPI_INFO_NULL; + acc_tpl = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(acc_tpl, comm_, info); +#else + assert(0); // should be unreachable +#endif + } + if (mode == OpenMode::OVERWRITE || (mode == OpenMode::APPEND && !exists)) { m_file = H5Fcreate(fileName.c_str(), H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT); + H5P_DEFAULT, acc_tpl); } else { m_file = H5Fopen(fileName.c_str(), mode == OpenMode::READ ? H5F_ACC_RDONLY : H5F_ACC_RDWR, - H5P_DEFAULT); + acc_tpl); } if (m_file == H5I_INVALID_HID) { throw std::runtime_error(std::string("HDF5File: Failed to ") + @@ -65,6 +81,10 @@ HDF5File::HDF5File(const std::string& fileName, OpenMode mode) (mode == OpenMode::APPEND && !exists) ? "create" : "open") + fileName); } + + if (comm_.size() > 1) { + H5Pclose(acc_tpl); + } } HDF5File::~HDF5File() @@ -76,63 +96,71 @@ HDF5File::~HDF5File() void HDF5File::write(const std::string& group, const std::string& dset, - const std::vector& buffer) + const std::vector& buffer, + DataSetMode mode) const { - hid_t grp; - if (groupExists(m_file, group)) { - grp = H5Gopen2(m_file, group.c_str(), H5P_DEFAULT); + hid_t grp = H5I_INVALID_HID; + std::string realGroup = group; + if (mode == DataSetMode::PROCESS_SPLIT) { + if (group != "/") + realGroup += '/'; + realGroup += dset; + } + + OPM_BEGIN_PARALLEL_TRY_CATCH(); + + if (groupExists(m_file, realGroup)) { + grp = H5Gopen2(m_file, realGroup.c_str(), H5P_DEFAULT); } else { - auto grps = split_string(group, '/'); + auto grps = split_string(realGroup, '/'); std::string curr; - for (size_t i = 0; i < grps.size()-1; ++i) { + for (size_t i = 0; i < grps.size(); ++i) { + if (grps[i].empty()) + continue; curr += '/'; curr += grps[i]; if (!groupExists(m_file, curr)) { hid_t subgrp = H5Gcreate2(m_file, curr.c_str(), 0, H5P_DEFAULT, H5P_DEFAULT); if (subgrp == H5I_INVALID_HID) { - throw std::runtime_error("HDF5File: Failed to create group '" + curr + "'"); + throw std::runtime_error("Failed to create group '" + curr + "'"); } - H5Gclose(subgrp); + if (i == grps.size() - 1) { + grp = subgrp; + } else { + H5Gclose(subgrp); + } + } else if (i == grps.size() - 1) { + grp = H5Gopen2(m_file, realGroup.c_str(), H5P_DEFAULT); } } - grp = H5Gcreate2(m_file, group.c_str(), 0, H5P_DEFAULT, H5P_DEFAULT); } if (grp == H5I_INVALID_HID) { - throw std::runtime_error("HDF5File: Failed to create group '" + group + "'"); + throw std::runtime_error("Failed to create group '" + realGroup + "'"); } - hsize_t size = buffer.size(); - hsize_t start = 0; - - hid_t space = H5Screate_simple(1, &size, nullptr); - hid_t dataset_id = H5Dcreate2(grp, dset.c_str(), H5T_NATIVE_CHAR, space, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dataset_id == H5I_INVALID_HID) { - throw std::runtime_error("HDF5File: Trying to write already existing dataset '" + group + '/' + dset + "'"); + if (mode == DataSetMode::PROCESS_SPLIT) { + writeSplit(grp, buffer, realGroup); + } else if (mode == DataSetMode::ROOT_ONLY) { + writeRootOnly(grp, buffer, group, dset); } - - if (size > 0) { - hid_t filespace = H5Dget_space(dataset_id); - hsize_t stride = 1; - H5Sselect_hyperslab(filespace, H5S_SELECT_SET, &start, &stride, &size, nullptr); - hid_t memspace = H5Screate_simple(1, &size, nullptr); - H5Dwrite(dataset_id, H5T_NATIVE_CHAR, memspace, filespace, H5P_DEFAULT, buffer.data()); - H5Sclose(memspace); - H5Sclose(filespace); - } - H5Dclose(dataset_id); - H5Sclose(space); H5Gclose(grp); + + OPM_END_PARALLEL_TRY_CATCH("HDF5File: Error writing data: ", comm_); } void HDF5File::read(const std::string& group, const std::string& dset, - std::vector& buffer) const + std::vector& buffer, + DataSetMode mode) const { - hid_t dataset_id = H5Dopen2(m_file, (group + "/"+ dset).c_str(), H5P_DEFAULT); + std::string realSet = group + '/' + dset; + if (mode == DataSetMode::PROCESS_SPLIT) { + realSet += '/' + std::to_string(comm_.rank()); + } + hid_t dataset_id = H5Dopen2(m_file, realSet.c_str(), H5P_DEFAULT); if (dataset_id == H5I_INVALID_HID) { - throw std::runtime_error("HDF5File: Trying to read non-existing dataset " + group + '/' + dset); + throw std::runtime_error("Trying to read non-existing dataset " + group + '/' + dset); } hid_t space = H5Dget_space(dataset_id); @@ -157,10 +185,79 @@ std::vector HDF5File::list(const std::string& group) const if (H5Literate_by_name(m_file, group.c_str(), H5_INDEX_NAME, H5_ITER_INC, &idx, list_group, &result, H5P_DEFAULT) < 0) { - throw std::runtime_error("Failure while listing HDF5 group '" + group + "'"); + throw std::runtime_error("Failure while listing group '" + group + "'"); } return result; } +void HDF5File::writeSplit(hid_t grp, + const std::vector& buffer, + const std::string& dset) const +{ + std::vector proc_sizes(comm_.size()); + if (comm_.size() > 1) { + hsize_t lsize = buffer.size(); + comm_.allgather(&lsize, 1, proc_sizes.data()); + } else { + proc_sizes[0] = buffer.size(); + } + + for (int i = 0; i < comm_.size(); ++i) { + hid_t space = H5Screate_simple(1, &proc_sizes[i], nullptr); + hid_t dataset_id = H5Dcreate2(grp, + std::to_string(i).c_str(), + H5T_NATIVE_CHAR, space, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dataset_id == H5I_INVALID_HID) { + throw std::runtime_error("Trying to write already existing dataset '" + + dset + '/' + std::to_string(i) + "'"); + } + + if (i == comm_.rank()) { + hid_t filespace = H5Dget_space(dataset_id); + hsize_t stride = 1; + hsize_t start = 0; + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, &start, &stride, &proc_sizes[i], nullptr); + hid_t memspace = H5Screate_simple(1, &proc_sizes[i], nullptr); + H5Dwrite(dataset_id, H5T_NATIVE_CHAR, memspace, filespace, H5P_DEFAULT, buffer.data()); + H5Sclose(memspace); + H5Sclose(filespace); + } + H5Dclose(dataset_id); + H5Sclose(space); + } +} + +void HDF5File::writeRootOnly(hid_t grp, + const std::vector& buffer, + const std::string& group, + const std::string& dset) const +{ + hsize_t size = buffer.size(); + comm_.broadcast(&size, 1, 0); + hid_t space = H5Screate_simple(1, &size, nullptr); + hid_t dataset_id = H5Dcreate2(grp, + dset.c_str(), + H5T_NATIVE_CHAR, space, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dataset_id == H5I_INVALID_HID) { + throw std::runtime_error("Trying to write already existing dataset '" + + group + '/' + dset + "'"); + } + + if (comm_.rank() == 0) { + hid_t filespace = H5Dget_space(dataset_id); + hsize_t stride = 1; + hsize_t start = 0; + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, &start, &stride, &size, nullptr); + hid_t memspace = H5Screate_simple(1, &size, nullptr); + H5Dwrite(dataset_id, H5T_NATIVE_CHAR, memspace, filespace, H5P_DEFAULT, buffer.data()); + H5Sclose(memspace); + H5Sclose(filespace); + } + H5Dclose(dataset_id); + H5Sclose(space); +} + } diff --git a/opm/simulators/utils/HDF5File.hpp b/opm/simulators/utils/HDF5File.hpp index 598f1d4a6..ea3a3e4cd 100644 --- a/opm/simulators/utils/HDF5File.hpp +++ b/opm/simulators/utils/HDF5File.hpp @@ -21,6 +21,8 @@ #ifndef HDF5_FILE_HPP #define HDF5_FILE_HPP +#include + #include #include @@ -38,10 +40,18 @@ public: READ //!< Open existing file for reading }; + //! \brief Enumeration of dataset modes. + enum class DataSetMode { + ROOT_ONLY, //!< A single dataset created at the root process + PROCESS_SPLIT //!< One separate data set for each parallel process + }; + //! \brief Opens HDF5 file for I/O. //! \param fileName Name of file to open //! \param mode Open mode for file - HDF5File(const std::string& fileName, OpenMode mode); + HDF5File(const std::string& fileName, + OpenMode mode, + Parallel::Communication comm); //! \brief Destructor clears up any opened files. ~HDF5File(); @@ -53,7 +63,8 @@ public: //! \details Throws exception on failure void write(const std::string& group, const std::string& dset, - const std::vector& buffer); + const std::vector& buffer, + DataSetMode mode = DataSetMode::PROCESS_SPLIT) const; //! \brief Read a char buffer from a specified location in file. //! \param group Group ("directory") to read data from @@ -62,14 +73,33 @@ public: //! \details Throws exception on failure void read(const std::string& group, const std::string& dset, - std::vector& buffer) const; + std::vector& buffer, + DataSetMode Mode = DataSetMode::PROCESS_SPLIT) const; //! \brief Lists the entries in a given group. //! \details Note: Both datasets and subgroups are returned std::vector list(const std::string& group) const; private: + //! \brief Write data from each process to a separate dataset. + //! \param grp Handle for group to store dataset in + //! \param buffer Data to write + //! \param dset Name of dataset + void writeSplit(hid_t grp, + const std::vector& buffer, + const std::string& dset) const; + + //! \brief Write data from root process only. + //! \param grp Handle for group to store dataset in + //! \param buffer Data to write + //! \param dset Name of dataset + void writeRootOnly(hid_t grp, + const std::vector& buffer, + const std::string& group, + const std::string& dset) const; + hid_t m_file = H5I_INVALID_HID; //!< File handle + Parallel::Communication comm_; }; } diff --git a/tests/test_HDF5File.cpp b/tests/test_HDF5File.cpp index 626665fe0..097bc9be3 100644 --- a/tests/test_HDF5File.cpp +++ b/tests/test_HDF5File.cpp @@ -24,6 +24,7 @@ #include #define BOOST_TEST_MODULE HDF5FileTest +#define BOOST_TEST_NO_MAIN #include #include @@ -36,13 +37,18 @@ BOOST_AUTO_TEST_CASE(ReadWrite) auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); std::filesystem::create_directory(path); auto rwpath = (path / "rw.hdf5").string(); +#if HAVE_MPI + Parallel::Communication comm(MPI_COMM_SELF); +#else + Parallel::Communcation comm; +#endif const std::vector test_data{1,2,3,4,5,6,8,9}; { - Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE); + Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm); BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data)); } { - Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ); + Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ, comm); std::vector data; BOOST_CHECK_NO_THROW(in_file.read("/test_data", "d1", data)); BOOST_CHECK_EQUAL_COLLECTIONS(data.begin(), data.end(), @@ -54,7 +60,12 @@ BOOST_AUTO_TEST_CASE(ReadWrite) BOOST_AUTO_TEST_CASE(ThrowOpenNonexistent) { - BOOST_CHECK_THROW(Opm::HDF5File out_file("no_such_file.hdf5", Opm::HDF5File::OpenMode::READ), std::runtime_error); +#if HAVE_MPI + Parallel::Communication comm(MPI_COMM_SELF); +#else + Parallel::Communcation comm; +#endif + BOOST_CHECK_THROW(Opm::HDF5File out_file("no_such_file.hdf5", Opm::HDF5File::OpenMode::READ, comm), std::runtime_error); } BOOST_AUTO_TEST_CASE(ReadNonExistentDset) @@ -62,13 +73,18 @@ BOOST_AUTO_TEST_CASE(ReadNonExistentDset) auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); std::filesystem::create_directory(path); auto rwpath = (path / "existent_dset.hdf5").string(); +#if HAVE_MPI + Parallel::Communication comm(MPI_COMM_SELF); +#else + Parallel::Communcation comm; +#endif const std::vector test_data{1,2,3,4,5,6,8,9}; { - Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE); + Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm); BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data)); } { - Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ); + Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ, comm); std::vector data; BOOST_CHECK_NO_THROW(in_file.read("/test_data", "d1", data)); BOOST_CHECK_EQUAL_COLLECTIONS(data.begin(), data.end(), @@ -84,9 +100,14 @@ BOOST_AUTO_TEST_CASE(WriteExistentDset) auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); std::filesystem::create_directory(path); auto rwpath = (path / "existent_dset.hdf5").string(); +#if HAVE_MPI + Parallel::Communication comm(MPI_COMM_SELF); +#else + Parallel::Communcation comm; +#endif const std::vector test_data{1,2,3,4,5,6,8,9}; { - Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE); + Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm); BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data)); BOOST_CHECK_THROW(out_file.write("/test_data", "d1", test_data), std::runtime_error); } @@ -99,15 +120,20 @@ BOOST_AUTO_TEST_CASE(List) auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); std::filesystem::create_directory(path); auto rwpath = (path / "existent_dset.hdf5").string(); +#if HAVE_MPI + Parallel::Communication comm(MPI_COMM_SELF); +#else + Parallel::Communcation comm; +#endif const std::vector test_data{1,2,3,4,5,6,8,9}; { - Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE); + Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm); BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data)); BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d2", test_data)); BOOST_CHECK_NO_THROW(out_file.write("/test_data/test", "d2", test_data)); } { - Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ); + Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ, comm); auto res1 = in_file.list("/"); BOOST_CHECK_EQUAL(res1.size(), 1u); @@ -124,3 +150,14 @@ BOOST_AUTO_TEST_CASE(List) std::filesystem::remove(rwpath); std::filesystem::remove(path); } + +bool init_unit_test_func() +{ + return true; +} + +int main(int argc, char** argv) +{ + Dune::MPIHelper::instance(argc, argv); + return boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv); +} diff --git a/tests/test_HDF5File_Parallel.cpp b/tests/test_HDF5File_Parallel.cpp new file mode 100644 index 000000000..6d92cba42 --- /dev/null +++ b/tests/test_HDF5File_Parallel.cpp @@ -0,0 +1,79 @@ +/* + Copyright 2021 Equinor. + + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ + +#include + +#include + +#include +#include + +#define BOOST_TEST_MODULE HDF5FileParallelTest +#define BOOST_TEST_NO_MAIN +#include + +#include +#include +#include +#include + +using namespace Opm; + +BOOST_AUTO_TEST_CASE(ReadWrite) +{ + std::string path; + Parallel::Communication comm; + if (comm.rank() == 0) { + path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); + } + std::size_t size = path.size(); + comm.broadcast(&size, 1, 0); + if (comm.rank() != 0) { + path.resize(size); + } + comm.broadcast(path.data(), size, 0); + std::filesystem::create_directory(path); + auto rwpath = (std::filesystem::path(path) / "rw.hdf5").string(); + std::vector test_data(10); + std::iota(test_data.begin(), test_data.end(), 10*comm.rank()); + { + Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm); + BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data)); + } + { + Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ, comm); + std::vector data; + BOOST_CHECK_NO_THROW(in_file.read("/test_data", "d1", data)); + BOOST_CHECK_EQUAL_COLLECTIONS(data.begin(), data.end(), + test_data.begin(), test_data.end()); + } + std::filesystem::remove(rwpath); + std::filesystem::remove(path); +} + +bool init_unit_test_func() +{ + return true; +} + +int main(int argc, char** argv) +{ + Dune::MPIHelper::instance(argc, argv); + return boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv); +} diff --git a/tests/test_HDF5Serializer.cpp b/tests/test_HDF5Serializer.cpp index 7f6fbe284..b0e765dbb 100644 --- a/tests/test_HDF5Serializer.cpp +++ b/tests/test_HDF5Serializer.cpp @@ -26,6 +26,7 @@ #include #define BOOST_TEST_MODULE HDF5FileTest +#define BOOST_TEST_NO_MAIN #include #include @@ -38,15 +39,21 @@ BOOST_AUTO_TEST_CASE(Header) auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); std::filesystem::create_directory(path); auto rwpath = (path / "rw.hdf5").string(); +#if HAVE_MPI + Parallel::Communication comm(MPI_COMM_SELF); +#else + Parallel::Communcation comm; +#endif std::array output{"foo", "bar", "foobar", "bob", "bobbar"}; { - HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE); + HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE, comm); ser.writeHeader(output[0], output[1], output[2], output[3], output[4], 5); } { - HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ); + HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ, comm); std::tuple,int> input; - ser.read(input, "/", "simulator_info"); + ser.read(input, "/", "simulator_info", + Opm::HDF5File::DataSetMode::ROOT_ONLY); const auto& [strings, num_procs] = input; BOOST_CHECK_EQUAL_COLLECTIONS(strings.begin(), strings.end(), output.begin(), output.end()); @@ -62,13 +69,18 @@ BOOST_AUTO_TEST_CASE(WriteRead) auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); std::filesystem::create_directory(path); auto rwpath = (path / "rw.hdf5").string(); +#if HAVE_MPI + Parallel::Communication comm(MPI_COMM_SELF); +#else + Parallel::Communcation comm; +#endif auto output = Group::serializationTestObject(); { - HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE); + HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE, comm); ser.write(output, "/report_step/10", "test"); } { - HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ); + HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ, comm); Group input; ser.read(input, "/report_step/10", "test"); BOOST_CHECK_MESSAGE(input == output, "Deserialized data does not match input"); @@ -81,3 +93,14 @@ BOOST_AUTO_TEST_CASE(WriteRead) std::filesystem::remove(rwpath); std::filesystem::remove(path); } + +bool init_unit_test_func() +{ + return true; +} + +int main(int argc, char** argv) +{ + Dune::MPIHelper::instance(argc, argv); + return boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv); +} diff --git a/tests/test_HDF5Serializer_Parallel.cpp b/tests/test_HDF5Serializer_Parallel.cpp new file mode 100644 index 000000000..cfe7f090a --- /dev/null +++ b/tests/test_HDF5Serializer_Parallel.cpp @@ -0,0 +1,120 @@ +/* + Copyright 2021 Equinor. + + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ + +#include + +#include + +#include + +#include + +#include + +#define BOOST_TEST_MODULE HDF5SerializerParallelTest +#define BOOST_TEST_NO_MAIN +#include + +#include +#include +#include + +using namespace Opm; + +BOOST_AUTO_TEST_CASE(Header) +{ + Parallel::Communication comm; + std::string path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); + if (comm.rank() == 0) { + path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); + } + std::size_t size = path.size(); + comm.broadcast(&size, 1, 0); + if (comm.rank() != 0) { + path.resize(size); + } + comm.broadcast(path.data(), size, 0); + std::filesystem::create_directory(path); + auto rwpath = (std::filesystem::path(path) / "rw.hdf5").string(); + std::array output{"foo", "bar", "foobar", "bob", "bobbar"}; + { + HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE, comm); + ser.writeHeader(output[0], output[1], output[2], + output[3], output[4], comm.size()); + } + { + HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ, comm); + std::tuple,int> input; + ser.read(input, "/", "simulator_info", + Opm::HDF5File::DataSetMode::ROOT_ONLY); + const auto& [strings, num_procs] = input; + BOOST_CHECK_EQUAL_COLLECTIONS(strings.begin(), strings.end(), + output.begin(), output.end()); + BOOST_CHECK_EQUAL(num_procs, comm.size()); + } + + std::filesystem::remove(rwpath); + std::filesystem::remove(path); +} + +BOOST_AUTO_TEST_CASE(WriteRead) +{ + Parallel::Communication comm; + std::string path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); + if (comm.rank() == 0) { + path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%"); + } + std::size_t size = path.size(); + comm.broadcast(&size, 1, 0); + if (comm.rank() != 0) { + path.resize(size); + } + comm.broadcast(path.data(), size, 0); + std::filesystem::create_directory(path); + auto rwpath = (std::filesystem::path(path) / "rw.hdf5").string(); + auto output = Group::serializationTestObject(); + { + HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE, comm); + ser.write(output, "/report_step/10", "test"); + } + { + HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ, comm); + Group input; + ser.read(input, "/report_step/10", "test"); + BOOST_CHECK_MESSAGE(input == output, "Deserialized data does not match input"); + BOOST_CHECK_EQUAL(ser.lastReportStep(), 10); + const auto steps = ser.reportSteps(); + BOOST_CHECK_EQUAL(steps.size(), 1u); + BOOST_CHECK_EQUAL(steps[0], 10); + } + + std::filesystem::remove(rwpath); + std::filesystem::remove(path); +} + +bool init_unit_test_func() +{ + return true; +} + +int main(int argc, char** argv) +{ + Dune::MPIHelper::instance(argc, argv); + return boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv); +}