added: parallel support to HDF5File / HDF5Serializer

CMakeLists.txt

@@ -91,6 +91,10 @@ endif()
 
 find_package(opm-common REQUIRED)
 
+if(USE_MPI)
+  set(HDF5_PREFER_PARALLEL TRUE)
+endif()
+
 include(OpmInit)
 OpmSetPolicies()
 
@@ -340,6 +344,30 @@ opm_add_test(test_broadcast
     -b ${PROJECT_BINARY_DIR}
 )
 
+opm_add_test(test_HDF5File_Parallel
+  DEPENDS "opmsimulators"
+  LIBRARIES opmsimulators ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}
+  SOURCES
+    tests/test_HDF5File_Parallel.cpp
+  CONDITION
+    HDF5_FOUND AND MPI_FOUND AND Boost_UNIT_TEST_FRAMEWORK_FOUND
+  DRIVER_ARGS
+    -n 4
+    -b ${PROJECT_BINARY_DIR}
+)
+
+opm_add_test(test_HDF5Serializer_Parallel
+  DEPENDS "opmsimulators"
+  LIBRARIES opmsimulators ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}
+  SOURCES
+    tests/test_HDF5Serializer_Parallel.cpp
+  CONDITION
+    HDF5_FOUND AND MPI_FOUND AND Boost_UNIT_TEST_FRAMEWORK_FOUND
+  DRIVER_ARGS
+    -n 4
+    -b ${PROJECT_BINARY_DIR}
+)
+
 include(OpmBashCompletion)
 
 if (NOT BUILD_FLOW)

ebos/hdf5serializer.hh

@@ -25,6 +25,7 @@
 
 #include <opm/simulators/utils/HDF5File.hpp>
 #include <opm/simulators/utils/moduleVersion.hpp>
+#include <opm/simulators/utils/ParallelCommunication.hpp>
 #include <opm/simulators/utils/SerializationPackers.hpp>
 
 #include <algorithm>
@@ -36,9 +37,11 @@ namespace Opm {
 //! \brief Class for (de-)serializing using HDF5.
 class HDF5Serializer : public Serializer<Serialization::MemPacker> {
 public:
-    HDF5Serializer(const std::string& fileName, HDF5File::OpenMode mode)
+    HDF5Serializer(const std::string& fileName,
+                   HDF5File::OpenMode mode,
+                   Parallel::Communication comm)
         : Serializer<Serialization::MemPacker>(m_packer_priv)
-        , m_h5file(fileName, mode)
+        , m_h5file(fileName, mode, comm)
     {}
 
     //! \brief Serialize and write data to restart file.
@@ -47,7 +50,8 @@ public:
     template<class T>
     void write(T& data,
                const std::string& group,
-               const std::string& dset)
+               const std::string& dset,
+               HDF5File::DataSetMode mode = HDF5File::DataSetMode::PROCESS_SPLIT)
     {
         try {
             this->pack(data);
@@ -56,7 +60,7 @@ public:
             throw;
         }
 
-        m_h5file.write(group, dset, m_buffer);
+        m_h5file.write(group, dset, m_buffer, mode);
     }
 
     //! \brief Writes a header to the file.
@@ -80,7 +84,7 @@ public:
             m_packSize = std::numeric_limits<size_t>::max();
             throw;
         }
-        m_h5file.write("/", "simulator_info", m_buffer);
+        m_h5file.write("/", "simulator_info", m_buffer, HDF5File::DataSetMode::ROOT_ONLY);
     }
 
     //! \brief Read data and deserialize from restart file.
@@ -89,9 +93,10 @@ public:
     template<class T>
     void read(T& data,
               const std::string& group,
-              const std::string& dset)
+              const std::string& dset,
+              HDF5File::DataSetMode mode = HDF5File::DataSetMode::PROCESS_SPLIT)
     {
-        m_h5file.read(group, dset, m_buffer);
+        m_h5file.read(group, dset, m_buffer, mode);
         this->unpack(data);
     }
 

examples/opmrst_inspect.cpp

@@ -22,6 +22,7 @@
 #include <ebos/hdf5serializer.hh>
 
 #include <opm/simulators/timestepping/SimulatorTimer.hpp>
+#include <opm/simulators/utils/ParallelCommunication.hpp>
 
 #include <boost/date_time.hpp>
 
@@ -38,7 +39,14 @@ int main(int argc, char** argv)
         return 1;
     }
 
-    Opm::HDF5Serializer ser(argv[1], Opm::HDF5File::OpenMode::READ);
+#if HAVE_MPI
+    Opm::Parallel::Communication comm(MPI_COMM_SELF);
+#else
+    Opm::Parallel::Communication comm();
+#endif
+
+    Dune::MPIHelper::instance(argc, argv);
+    Opm::HDF5Serializer ser(argv[1], Opm::HDF5File::OpenMode::READ, comm);
 
     std::tuple<std::array<std::string,5>,int> header;
     try {

opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp

@@ -569,8 +569,9 @@ protected:
             return;
         }
 
-        int nextStep = timer.currentStepNum();
+        OPM_BEGIN_PARALLEL_TRY_CATCH();
 
+        int nextStep = timer.currentStepNum();
         if ((saveStep_ != -1 && nextStep == saveStep_)  ||
             (saveStride_ != -1 && (nextStep % saveStride_) == 0)) {
 #if !HAVE_HDF5
@@ -580,7 +581,9 @@ protected:
             if (nextStep == saveStride_ || nextStep == saveStep_) {
                 std::filesystem::remove(saveFile_);
             }
-            HDF5Serializer writer(saveFile_, HDF5File::OpenMode::APPEND);
+            HDF5Serializer writer(saveFile_,
+                                  HDF5File::OpenMode::APPEND,
+                                  EclGenericVanguard::comm());
             if (nextStep == saveStride_ || nextStep == saveStep_) {
                 std::ostringstream str;
                 Parameters::printValues<TypeTag>(str);
@@ -592,10 +595,14 @@ protected:
                                    EclGenericVanguard::comm().size());
             }
             writer.write(*this, groupName, "simulator_data");
-            writer.write(timer, groupName, "simulator_timer");
+            writer.write(timer, groupName, "simulator_timer",
+                         HDF5File::DataSetMode::ROOT_ONLY);
             OpmLog::info("Serialized state written for report step " + std::to_string(nextStep));
 #endif
         }
+
+        OPM_END_PARALLEL_TRY_CATCH("Error saving serialized state: ",
+                                   EclGenericVanguard::comm());
     }
 
     //! \brief Load timer info from serialized state.
@@ -605,13 +612,22 @@ protected:
         OpmLog::error("Loading of serialized state requested, but no HDF5 support available.");
         loadStep_ = -1;
 #else
-        HDF5Serializer reader(saveFile_, HDF5File::OpenMode::READ);
+        OPM_BEGIN_PARALLEL_TRY_CATCH();
-        if (loadStep_ == 0)
+
+        HDF5Serializer reader(saveFile_,
+                              HDF5File::OpenMode::READ,
+                              EclGenericVanguard::comm());
+
+        if (loadStep_ == 0) {
             loadStep_ = reader.lastReportStep();
+        }
 
         OpmLog::info("Loading serialized state for report step " + std::to_string(loadStep_));
         const std::string groupName = "/report_step/" + std::to_string(loadStep_);
-        reader.read(timer, groupName, "simulator_timer");
+        reader.read(timer, groupName, "simulator_timer", HDF5File::DataSetMode::ROOT_ONLY);
+
+        OPM_END_PARALLEL_TRY_CATCH("Error loading serialized state: ",
+                                   EclGenericVanguard::comm());
 #endif
     }
 
@@ -619,9 +635,16 @@ protected:
     void loadSimulatorState()
     {
 #if HAVE_HDF5
-        HDF5Serializer reader(saveFile_, HDF5File::OpenMode::READ);
+        OPM_BEGIN_PARALLEL_TRY_CATCH();
+
+        HDF5Serializer reader(saveFile_,
+                              HDF5File::OpenMode::READ,
+                              EclGenericVanguard::comm());
         const std::string groupName = "/report_step/" + std::to_string(loadStep_);
         reader.read(*this, groupName, "simulator_data");
+
+        OPM_END_PARALLEL_TRY_CATCH("Error loading serialized state: ",
+                                   EclGenericVanguard::comm());
 #endif
     }
 

opm/simulators/utils/HDF5File.cpp

@@ -24,6 +24,8 @@
 
 #include <opm/common/utility/String.hpp>
 
+#include <opm/simulators/utils/DeferredLoggingErrorHelpers.hpp>
+
 #include <filesystem>
 #include <stdexcept>
 
@@ -46,18 +48,32 @@ bool groupExists(hid_t parent, const std::string& path)
 
 namespace Opm {
 
-HDF5File::HDF5File(const std::string& fileName, OpenMode mode)
+HDF5File::HDF5File(const std::string& fileName,
+                   OpenMode mode,
+                   Parallel::Communication comm)
+    : comm_(comm)
 {
     bool exists = std::filesystem::exists(fileName);
+    hid_t acc_tpl = H5P_DEFAULT;
+    if (comm.size() > 1) {
+#if HAVE_MPI
+        MPI_Info info = MPI_INFO_NULL;
+        acc_tpl = H5Pcreate(H5P_FILE_ACCESS);
+        H5Pset_fapl_mpio(acc_tpl, comm_, info);
+#else
+        assert(0); // should be unreachable
+#endif
+    }
+
     if (mode == OpenMode::OVERWRITE ||
         (mode == OpenMode::APPEND && !exists)) {
         m_file = H5Fcreate(fileName.c_str(),
                            H5F_ACC_TRUNC,
-                           H5P_DEFAULT, H5P_DEFAULT);
+                           H5P_DEFAULT, acc_tpl);
     } else {
         m_file = H5Fopen(fileName.c_str(),
                          mode == OpenMode::READ ? H5F_ACC_RDONLY : H5F_ACC_RDWR,
-                         H5P_DEFAULT);
+                         acc_tpl);
     }
     if (m_file == H5I_INVALID_HID) {
         throw std::runtime_error(std::string("HDF5File: Failed to ") +
@@ -65,6 +81,10 @@ HDF5File::HDF5File(const std::string& fileName, OpenMode mode)
                                   (mode == OpenMode::APPEND && !exists) ? "create" : "open") +
                                   fileName);
     }
+
+    if (comm_.size() > 1) {
+        H5Pclose(acc_tpl);
+    }
 }
 
 HDF5File::~HDF5File()
@@ -76,63 +96,71 @@ HDF5File::~HDF5File()
 
 void HDF5File::write(const std::string& group,
                      const std::string& dset,
-                     const std::vector<char>& buffer)
+                     const std::vector<char>& buffer,
+                     DataSetMode mode) const
 {
-    hid_t grp;
+    hid_t grp = H5I_INVALID_HID;
-    if (groupExists(m_file, group)) {
+    std::string realGroup = group;
-        grp = H5Gopen2(m_file, group.c_str(), H5P_DEFAULT);
+    if (mode == DataSetMode::PROCESS_SPLIT) {
+        if (group != "/")
+            realGroup += '/';
+        realGroup += dset;
+    }
+
+    OPM_BEGIN_PARALLEL_TRY_CATCH();
+
+    if (groupExists(m_file, realGroup)) {
+        grp = H5Gopen2(m_file, realGroup.c_str(), H5P_DEFAULT);
     } else {
-        auto grps = split_string(group, '/');
+        auto grps = split_string(realGroup, '/');
         std::string curr;
-        for (size_t i = 0; i < grps.size()-1; ++i) {
+        for (size_t i = 0; i < grps.size(); ++i) {
+            if (grps[i].empty())
+                continue;
             curr += '/';
             curr += grps[i];
             if (!groupExists(m_file, curr)) {
                 hid_t subgrp = H5Gcreate2(m_file, curr.c_str(), 0, H5P_DEFAULT, H5P_DEFAULT);
                 if (subgrp == H5I_INVALID_HID) {
-                    throw std::runtime_error("HDF5File: Failed to create group '" + curr + "'");
+                    throw std::runtime_error("Failed to create group '" + curr + "'");
                 }
-                H5Gclose(subgrp);
+                if (i == grps.size() - 1) {
+                    grp = subgrp;
+                } else {
+                    H5Gclose(subgrp);
+                }
+            } else if (i == grps.size() - 1) {
+                grp = H5Gopen2(m_file, realGroup.c_str(), H5P_DEFAULT);
             }
         }
-        grp = H5Gcreate2(m_file, group.c_str(), 0, H5P_DEFAULT, H5P_DEFAULT);
     }
 
     if (grp == H5I_INVALID_HID) {
-        throw std::runtime_error("HDF5File: Failed to create group '" + group + "'");
+        throw std::runtime_error("Failed to create group '" + realGroup + "'");
     }
 
-    hsize_t size = buffer.size();
+    if (mode == DataSetMode::PROCESS_SPLIT) {
-    hsize_t start = 0;
+        writeSplit(grp, buffer, realGroup);
-
+    } else if (mode == DataSetMode::ROOT_ONLY) {
-    hid_t space = H5Screate_simple(1, &size, nullptr);
+        writeRootOnly(grp, buffer, group, dset);
-    hid_t dataset_id = H5Dcreate2(grp, dset.c_str(), H5T_NATIVE_CHAR, space,
-                       H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
-    if (dataset_id == H5I_INVALID_HID) {
-        throw std::runtime_error("HDF5File: Trying to write already existing dataset '" + group + '/' + dset + "'");
     }
-
-    if (size > 0) {
-        hid_t filespace = H5Dget_space(dataset_id);
-        hsize_t stride = 1;
-        H5Sselect_hyperslab(filespace, H5S_SELECT_SET, &start, &stride, &size, nullptr);
-        hid_t memspace = H5Screate_simple(1, &size, nullptr);
-        H5Dwrite(dataset_id, H5T_NATIVE_CHAR, memspace, filespace, H5P_DEFAULT, buffer.data());
-        H5Sclose(memspace);
-        H5Sclose(filespace);
-    }
-    H5Dclose(dataset_id);
-    H5Sclose(space);
     H5Gclose(grp);
+
+    OPM_END_PARALLEL_TRY_CATCH("HDF5File: Error writing data: ", comm_);
 }
 
 void HDF5File::read(const std::string& group,
                     const std::string& dset,
-                    std::vector<char>& buffer) const
+                    std::vector<char>& buffer,
+                    DataSetMode mode) const
 {
-    hid_t dataset_id = H5Dopen2(m_file, (group + "/"+ dset).c_str(), H5P_DEFAULT);
+    std::string realSet = group + '/' + dset;
+    if (mode == DataSetMode::PROCESS_SPLIT) {
+        realSet += '/' + std::to_string(comm_.rank());
+    }
+    hid_t dataset_id = H5Dopen2(m_file, realSet.c_str(), H5P_DEFAULT);
     if (dataset_id == H5I_INVALID_HID) {
-        throw std::runtime_error("HDF5File: Trying to read non-existing dataset " + group + '/' + dset);
+        throw std::runtime_error("Trying to read non-existing dataset " + group + '/' + dset);
     }
 
     hid_t space = H5Dget_space(dataset_id);
@@ -157,10 +185,79 @@ std::vector<std::string> HDF5File::list(const std::string& group) const
     if (H5Literate_by_name(m_file, group.c_str(),
                            H5_INDEX_NAME, H5_ITER_INC,
                            &idx, list_group, &result, H5P_DEFAULT) < 0) {
-        throw std::runtime_error("Failure while listing HDF5 group '" + group + "'");
+        throw std::runtime_error("Failure while listing group '" + group + "'");
     }
 
     return result;
 }
 
+void HDF5File::writeSplit(hid_t grp,
+                          const std::vector<char>& buffer,
+                          const std::string& dset) const
+{
+    std::vector<hsize_t> proc_sizes(comm_.size());
+    if (comm_.size() > 1) {
+        hsize_t lsize = buffer.size();
+        comm_.allgather(&lsize, 1, proc_sizes.data());
+    } else {
+        proc_sizes[0] = buffer.size();
+    }
+
+    for (int i = 0; i < comm_.size(); ++i) {
+        hid_t space = H5Screate_simple(1, &proc_sizes[i], nullptr);
+        hid_t dataset_id = H5Dcreate2(grp,
+                                      std::to_string(i).c_str(),
+                                      H5T_NATIVE_CHAR, space,
+                                      H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+        if (dataset_id == H5I_INVALID_HID) {
+            throw std::runtime_error("Trying to write already existing dataset '" +
+                                     dset + '/' + std::to_string(i) + "'");
+        }
+
+        if (i == comm_.rank()) {
+            hid_t filespace = H5Dget_space(dataset_id);
+            hsize_t stride = 1;
+            hsize_t start = 0;
+            H5Sselect_hyperslab(filespace, H5S_SELECT_SET, &start, &stride, &proc_sizes[i], nullptr);
+            hid_t memspace = H5Screate_simple(1, &proc_sizes[i], nullptr);
+            H5Dwrite(dataset_id, H5T_NATIVE_CHAR, memspace, filespace, H5P_DEFAULT, buffer.data());
+            H5Sclose(memspace);
+            H5Sclose(filespace);
+        }
+        H5Dclose(dataset_id);
+        H5Sclose(space);
+    }
+}
+
+void HDF5File::writeRootOnly(hid_t grp,
+                             const std::vector<char>& buffer,
+                             const std::string& group,
+                             const std::string& dset) const
+{
+    hsize_t size = buffer.size();
+    comm_.broadcast(&size, 1, 0);
+    hid_t space = H5Screate_simple(1, &size, nullptr);
+    hid_t dataset_id = H5Dcreate2(grp,
+                                  dset.c_str(),
+                                  H5T_NATIVE_CHAR, space,
+                                  H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    if (dataset_id == H5I_INVALID_HID) {
+        throw std::runtime_error("Trying to write already existing dataset '" +
+                                 group + '/' + dset + "'");
+    }
+
+    if (comm_.rank() == 0) {
+        hid_t filespace = H5Dget_space(dataset_id);
+        hsize_t stride = 1;
+        hsize_t start = 0;
+        H5Sselect_hyperslab(filespace, H5S_SELECT_SET, &start, &stride, &size, nullptr);
+        hid_t memspace = H5Screate_simple(1, &size, nullptr);
+        H5Dwrite(dataset_id, H5T_NATIVE_CHAR, memspace, filespace, H5P_DEFAULT, buffer.data());
+        H5Sclose(memspace);
+        H5Sclose(filespace);
+    }
+    H5Dclose(dataset_id);
+    H5Sclose(space);
+}
+
 }

opm/simulators/utils/HDF5File.hpp

@@ -21,6 +21,8 @@
 #ifndef HDF5_FILE_HPP
 #define HDF5_FILE_HPP
 
+#include <opm/simulators/utils/ParallelCommunication.hpp>
+
 #include <hdf5.h>
 
 #include <string>
@@ -38,10 +40,18 @@ public:
         READ       //!< Open existing file for reading
     };
 
+    //! \brief Enumeration of dataset modes.
+    enum class DataSetMode {
+        ROOT_ONLY,     //!< A single dataset created at the root process
+        PROCESS_SPLIT  //!< One separate data set for each parallel process
+    };
+
     //! \brief Opens HDF5 file for I/O.
     //! \param fileName Name of file to open
     //! \param mode Open mode for file
-    HDF5File(const std::string& fileName, OpenMode mode);
+    HDF5File(const std::string& fileName,
+             OpenMode mode,
+             Parallel::Communication comm);
 
     //! \brief Destructor clears up any opened files.
     ~HDF5File();
@@ -53,7 +63,8 @@ public:
     //! \details Throws exception on failure
     void write(const std::string& group,
                const std::string& dset,
-               const std::vector<char>& buffer);
+               const std::vector<char>& buffer,
+               DataSetMode mode = DataSetMode::PROCESS_SPLIT) const;
 
     //! \brief Read a char buffer from a specified location in file.
     //! \param group Group ("directory") to read data from
@@ -62,14 +73,33 @@ public:
     //! \details Throws exception on failure
     void read(const std::string& group,
               const std::string& dset,
-              std::vector<char>& buffer) const;
+              std::vector<char>& buffer,
+              DataSetMode Mode = DataSetMode::PROCESS_SPLIT) const;
 
     //! \brief Lists the entries in a given group.
     //! \details Note: Both datasets and subgroups are returned
     std::vector<std::string> list(const std::string& group) const;
 
 private:
+    //! \brief Write data from each process to a separate dataset.
+    //! \param grp Handle for group to store dataset in
+    //! \param buffer Data to write
+    //! \param dset Name of dataset
+    void writeSplit(hid_t grp,
+                    const std::vector<char>& buffer,
+                    const std::string& dset) const;
+
+    //! \brief Write data from root process only.
+    //! \param grp Handle for group to store dataset in
+    //! \param buffer Data to write
+    //! \param dset Name of dataset
+    void writeRootOnly(hid_t grp,
+                       const std::vector<char>& buffer,
+                       const std::string& group,
+                       const std::string& dset) const;
+
     hid_t m_file = H5I_INVALID_HID; //!< File handle
+    Parallel::Communication comm_;
 };
 
 }

tests/test_HDF5File.cpp

@@ -24,6 +24,7 @@
 #include <opm/simulators/utils/HDF5File.hpp>
 
 #define BOOST_TEST_MODULE HDF5FileTest
+#define BOOST_TEST_NO_MAIN
 #include <boost/test/unit_test.hpp>
 
 #include <filesystem>
@@ -36,13 +37,18 @@ BOOST_AUTO_TEST_CASE(ReadWrite)
     auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
     std::filesystem::create_directory(path);
     auto rwpath = (path / "rw.hdf5").string();
+#if HAVE_MPI
+    Parallel::Communication comm(MPI_COMM_SELF);
+#else
+    Parallel::Communcation comm;
+#endif
     const std::vector<char> test_data{1,2,3,4,5,6,8,9};
     {
-        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE);
+        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm);
         BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data));
     }
     {
-        Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ);
+        Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ, comm);
         std::vector<char> data;
         BOOST_CHECK_NO_THROW(in_file.read("/test_data", "d1", data));
         BOOST_CHECK_EQUAL_COLLECTIONS(data.begin(), data.end(),
@@ -54,7 +60,12 @@ BOOST_AUTO_TEST_CASE(ReadWrite)
 
 BOOST_AUTO_TEST_CASE(ThrowOpenNonexistent)
 {
-    BOOST_CHECK_THROW(Opm::HDF5File out_file("no_such_file.hdf5", Opm::HDF5File::OpenMode::READ), std::runtime_error);
+#if HAVE_MPI
+    Parallel::Communication comm(MPI_COMM_SELF);
+#else
+    Parallel::Communcation comm;
+#endif
+    BOOST_CHECK_THROW(Opm::HDF5File out_file("no_such_file.hdf5", Opm::HDF5File::OpenMode::READ, comm), std::runtime_error);
 }
 
 BOOST_AUTO_TEST_CASE(ReadNonExistentDset)
@@ -62,13 +73,18 @@ BOOST_AUTO_TEST_CASE(ReadNonExistentDset)
     auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
     std::filesystem::create_directory(path);
     auto rwpath = (path / "existent_dset.hdf5").string();
+#if HAVE_MPI
+    Parallel::Communication comm(MPI_COMM_SELF);
+#else
+    Parallel::Communcation comm;
+#endif
     const std::vector<char> test_data{1,2,3,4,5,6,8,9};
     {
-        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE);
+        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm);
         BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data));
     }
     {
-        Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ);
+        Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ, comm);
         std::vector<char> data;
         BOOST_CHECK_NO_THROW(in_file.read("/test_data", "d1", data));
         BOOST_CHECK_EQUAL_COLLECTIONS(data.begin(), data.end(),
@@ -84,9 +100,14 @@ BOOST_AUTO_TEST_CASE(WriteExistentDset)
     auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
     std::filesystem::create_directory(path);
     auto rwpath = (path / "existent_dset.hdf5").string();
+#if HAVE_MPI
+    Parallel::Communication comm(MPI_COMM_SELF);
+#else
+    Parallel::Communcation comm;
+#endif
     const std::vector<char> test_data{1,2,3,4,5,6,8,9};
     {
-        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE);
+        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm);
         BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data));
         BOOST_CHECK_THROW(out_file.write("/test_data", "d1", test_data), std::runtime_error);
     }
@@ -99,15 +120,20 @@ BOOST_AUTO_TEST_CASE(List)
     auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
     std::filesystem::create_directory(path);
     auto rwpath = (path / "existent_dset.hdf5").string();
+#if HAVE_MPI
+    Parallel::Communication comm(MPI_COMM_SELF);
+#else
+    Parallel::Communcation comm;
+#endif
     const std::vector<char> test_data{1,2,3,4,5,6,8,9};
     {
-        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE);
+        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm);
         BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data));
         BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d2", test_data));
         BOOST_CHECK_NO_THROW(out_file.write("/test_data/test", "d2", test_data));
     }
     {
-        Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ);
+        Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ, comm);
 
         auto res1 = in_file.list("/");
         BOOST_CHECK_EQUAL(res1.size(), 1u);
@@ -124,3 +150,14 @@ BOOST_AUTO_TEST_CASE(List)
     std::filesystem::remove(rwpath);
     std::filesystem::remove(path);
 }
+
+bool init_unit_test_func()
+{
+    return true;
+}
+
+int main(int argc, char** argv)
+{
+    Dune::MPIHelper::instance(argc, argv);
+    return boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv);
+}

tests/test_HDF5File_Parallel.cpp

@@ -0,0 +1,79 @@
+/*
+  Copyright 2021 Equinor.
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+
+#include <opm/common/utility/FileSystem.hpp>
+
+#include <opm/simulators/utils/HDF5File.hpp>
+#include <opm/simulators/utils/ParallelCommunication.hpp>
+
+#define BOOST_TEST_MODULE HDF5FileParallelTest
+#define BOOST_TEST_NO_MAIN
+#include <boost/test/unit_test.hpp>
+
+#include <filesystem>
+#include <numeric>
+#include <stdexcept>
+#include <string>
+
+using namespace Opm;
+
+BOOST_AUTO_TEST_CASE(ReadWrite)
+{
+    std::string path;
+    Parallel::Communication comm;
+    if (comm.rank() == 0) {
+        path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
+    }
+    std::size_t size = path.size();
+    comm.broadcast(&size, 1, 0);
+    if (comm.rank() != 0) {
+        path.resize(size);
+    }
+    comm.broadcast(path.data(), size, 0);
+    std::filesystem::create_directory(path);
+    auto rwpath = (std::filesystem::path(path) / "rw.hdf5").string();
+    std::vector<char> test_data(10);
+    std::iota(test_data.begin(), test_data.end(), 10*comm.rank());
+    {
+        Opm::HDF5File out_file(rwpath, Opm::HDF5File::OpenMode::OVERWRITE, comm);
+        BOOST_CHECK_NO_THROW(out_file.write("/test_data", "d1", test_data));
+    }
+    {
+        Opm::HDF5File in_file(rwpath, Opm::HDF5File::OpenMode::READ, comm);
+        std::vector<char> data;
+        BOOST_CHECK_NO_THROW(in_file.read("/test_data", "d1", data));
+        BOOST_CHECK_EQUAL_COLLECTIONS(data.begin(), data.end(),
+                                      test_data.begin(), test_data.end());
+    }
+    std::filesystem::remove(rwpath);
+    std::filesystem::remove(path);
+}
+
+bool init_unit_test_func()
+{
+    return true;
+}
+
+int main(int argc, char** argv)
+{
+    Dune::MPIHelper::instance(argc, argv);
+    return boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv);
+}

tests/test_HDF5Serializer.cpp

@@ -26,6 +26,7 @@
 #include <opm/input/eclipse/Schedule/Group/Group.hpp>
 
 #define BOOST_TEST_MODULE HDF5FileTest
+#define BOOST_TEST_NO_MAIN
 #include <boost/test/unit_test.hpp>
 
 #include <filesystem>
@@ -38,15 +39,21 @@ BOOST_AUTO_TEST_CASE(Header)
     auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
     std::filesystem::create_directory(path);
     auto rwpath = (path / "rw.hdf5").string();
+#if HAVE_MPI
+    Parallel::Communication comm(MPI_COMM_SELF);
+#else
+    Parallel::Communcation comm;
+#endif
     std::array<std::string,5> output{"foo", "bar", "foobar", "bob", "bobbar"};
     {
-        HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE);
+        HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE, comm);
         ser.writeHeader(output[0], output[1], output[2], output[3], output[4], 5);
     }
     {
-        HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ);
+        HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ, comm);
         std::tuple<std::array<std::string,5>,int> input;
-        ser.read(input, "/", "simulator_info");
+        ser.read(input, "/", "simulator_info",
+                 Opm::HDF5File::DataSetMode::ROOT_ONLY);
         const auto& [strings, num_procs] = input;
         BOOST_CHECK_EQUAL_COLLECTIONS(strings.begin(), strings.end(),
                                       output.begin(), output.end());
@@ -62,13 +69,18 @@ BOOST_AUTO_TEST_CASE(WriteRead)
     auto path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
     std::filesystem::create_directory(path);
     auto rwpath = (path / "rw.hdf5").string();
+#if HAVE_MPI
+    Parallel::Communication comm(MPI_COMM_SELF);
+#else
+    Parallel::Communcation comm;
+#endif
     auto output = Group::serializationTestObject();
     {
-        HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE);
+        HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE, comm);
         ser.write(output, "/report_step/10", "test");
     }
     {
-        HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ);
+        HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ, comm);
         Group input;
         ser.read(input, "/report_step/10", "test");
         BOOST_CHECK_MESSAGE(input == output, "Deserialized data does not match input");
@@ -81,3 +93,14 @@ BOOST_AUTO_TEST_CASE(WriteRead)
     std::filesystem::remove(rwpath);
     std::filesystem::remove(path);
 }
+
+bool init_unit_test_func()
+{
+    return true;
+}
+
+int main(int argc, char** argv)
+{
+    Dune::MPIHelper::instance(argc, argv);
+    return boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv);
+}

tests/test_HDF5Serializer_Parallel.cpp

@@ -0,0 +1,120 @@
+/*
+  Copyright 2021 Equinor.
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+
+#include <ebos/hdf5serializer.hh>
+
+#include <opm/common/utility/FileSystem.hpp>
+
+#include <opm/input/eclipse/Schedule/Group/Group.hpp>
+
+#include <opm/simulators/utils/ParallelCommunication.hpp>
+
+#define BOOST_TEST_MODULE HDF5SerializerParallelTest
+#define BOOST_TEST_NO_MAIN
+#include <boost/test/unit_test.hpp>
+
+#include <filesystem>
+#include <stdexcept>
+#include <string>
+
+using namespace Opm;
+
+BOOST_AUTO_TEST_CASE(Header)
+{
+    Parallel::Communication comm;
+    std::string path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
+    if (comm.rank() == 0) {
+        path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
+    }
+    std::size_t size = path.size();
+    comm.broadcast(&size, 1, 0);
+    if (comm.rank() != 0) {
+        path.resize(size);
+    }
+    comm.broadcast(path.data(), size, 0);
+    std::filesystem::create_directory(path);
+    auto rwpath = (std::filesystem::path(path) / "rw.hdf5").string();
+    std::array<std::string,5> output{"foo", "bar", "foobar", "bob", "bobbar"};
+    {
+        HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE, comm);
+        ser.writeHeader(output[0], output[1], output[2],
+                        output[3], output[4], comm.size());
+    }
+    {
+        HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ, comm);
+        std::tuple<std::array<std::string,5>,int> input;
+        ser.read(input, "/", "simulator_info",
+                 Opm::HDF5File::DataSetMode::ROOT_ONLY);
+        const auto& [strings, num_procs] = input;
+        BOOST_CHECK_EQUAL_COLLECTIONS(strings.begin(), strings.end(),
+                                      output.begin(), output.end());
+        BOOST_CHECK_EQUAL(num_procs, comm.size());
+    }
+
+    std::filesystem::remove(rwpath);
+    std::filesystem::remove(path);
+}
+
+BOOST_AUTO_TEST_CASE(WriteRead)
+{
+    Parallel::Communication comm;
+    std::string path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
+    if (comm.rank() == 0) {
+        path = std::filesystem::temp_directory_path() / Opm::unique_path("hdf5test%%%%%");
+    }
+    std::size_t size = path.size();
+    comm.broadcast(&size, 1, 0);
+    if (comm.rank() != 0) {
+        path.resize(size);
+    }
+    comm.broadcast(path.data(), size, 0);
+    std::filesystem::create_directory(path);
+    auto rwpath = (std::filesystem::path(path) / "rw.hdf5").string();
+    auto output = Group::serializationTestObject();
+    {
+        HDF5Serializer ser(rwpath, HDF5File::OpenMode::OVERWRITE, comm);
+        ser.write(output, "/report_step/10", "test");
+    }
+    {
+        HDF5Serializer ser(rwpath, HDF5File::OpenMode::READ, comm);
+        Group input;
+        ser.read(input, "/report_step/10", "test");
+        BOOST_CHECK_MESSAGE(input == output, "Deserialized data does not match input");
+        BOOST_CHECK_EQUAL(ser.lastReportStep(), 10);
+        const auto steps = ser.reportSteps();
+        BOOST_CHECK_EQUAL(steps.size(), 1u);
+        BOOST_CHECK_EQUAL(steps[0], 10);
+    }
+
+    std::filesystem::remove(rwpath);
+    std::filesystem::remove(path);
+}
+
+bool init_unit_test_func()
+{
+    return true;
+}
+
+int main(int argc, char** argv)
+{
+    Dune::MPIHelper::instance(argc, argv);
+    return boost::unit_test::unit_test_main(&init_unit_test_func, argc, argv);
+}