From 1433bc7a5540a479d4068ebf935b3938a302e99e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Skille?= Date: Tue, 17 May 2022 17:26:56 +0200 Subject: [PATCH] ExtESmry - support loading when number of time steps have increased (on disk) The ESMRY file can be updated with more time step data in the time span between opening an ESMRY file and loading actual summary data from the file. This can happen if the ESMRY is generated by an active run. Length of summary vectors extracted will be equal to number of time steps on disk when the file was first opened (with the constructor). --- opm/io/eclipse/ExtESmry.hpp | 12 +- src/opm/io/eclipse/ExtESmry.cpp | 303 ++++++++++++++++++++++---------- 2 files changed, 219 insertions(+), 96 deletions(-) diff --git a/opm/io/eclipse/ExtESmry.hpp b/opm/io/eclipse/ExtESmry.hpp index 4869eaa58..c0e6a0d7d 100644 --- a/opm/io/eclipse/ExtESmry.hpp +++ b/opm/io/eclipse/ExtESmry.hpp @@ -37,7 +37,7 @@ using TimeStepEntry = std::tuple; using RstEntry = std::tuple; // start, rstart + rstnum, keycheck, units, rstep, tstep -using LodsmryHeadType = std::tuple, std::vector, +using ExtSmryHeadType = std::tuple, std::vector, std::vector, std::vector>; class ExtESmry @@ -72,7 +72,7 @@ public: private: std::filesystem::path m_inputFileName; - std::vector m_lodsmry_files; + std::vector m_esmry_files; bool m_loadBaseRun; std::vector> m_keyword_index; @@ -91,15 +91,17 @@ private: size_t m_nTstep; std::vector m_seqIndex; - std::vector m_lod_offset; - std::vector m_lod_arr_size; + std::vector m_rstep_offset; time_point m_startdat; double m_io_opening; double m_io_loading; - uint64_t open_esmry(std::filesystem::path& inputFileName, LodsmryHeadType& lodsmry_head); + bool open_esmry(const std::filesystem::path& inputFileName, ExtSmryHeadType& ext_smry_head, uint64_t& rstep_offset); + + bool load_esmry(const std::vector& stringVect, const std::vector& keyIndexVect, + const std::vector& loadKeyIndex, int ind, int to_ind ); void updatePathAndRootName(std::filesystem::path& dir, std::filesystem::path& rootN); }; diff --git a/src/opm/io/eclipse/ExtESmry.cpp b/src/opm/io/eclipse/ExtESmry.cpp index a0fc86dc0..bce2f2595 100644 --- a/src/opm/io/eclipse/ExtESmry.cpp +++ b/src/opm/io/eclipse/ExtESmry.cpp @@ -38,6 +38,7 @@ #include #include #include +#include namespace { @@ -104,7 +105,7 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) : if (m_inputFileName.extension()!=".ESMRY") throw std::invalid_argument("Input file should have extension .ESMRY"); - m_lodsmry_files.push_back(m_inputFileName); + m_esmry_files.push_back(m_inputFileName); std::filesystem::path rootName = m_inputFileName.parent_path() / m_inputFileName.stem(); std::filesystem::path path = std::filesystem::current_path(); @@ -113,18 +114,29 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) : updatePathAndRootName(path, rootName); - LodsmryHeadType lodsmry_head; + ExtSmryHeadType ext_esmry_head; - auto lod_offset = open_esmry(m_inputFileName, lodsmry_head); + uint64_t rstep_offset; - m_startdat = std::get<0>(lodsmry_head); + bool res = open_esmry(m_inputFileName, ext_esmry_head, rstep_offset); + int n_attempts = 1; - m_lod_offset.push_back(lod_offset); + while ((!res) && (n_attempts < 10)){ + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + res = open_esmry(m_inputFileName, ext_esmry_head, rstep_offset); + n_attempts ++; + } + + if (n_attempts == 10) + OPM_THROW( std::runtime_error, "when opening ESMRY file " + filename ); + + m_startdat = std::get<0>(ext_esmry_head); + m_rstep_offset.push_back(rstep_offset); std::map key_index; - auto keyword = std::get<2>(lodsmry_head); - auto units = std::get<3>(lodsmry_head); + auto keyword = std::get<2>(ext_esmry_head); + auto units = std::get<3>(ext_esmry_head); for (size_t n = 0; n < keyword.size(); n++){ key_index[keyword[n]] = n; @@ -136,17 +148,13 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) : for (size_t n = 0; n < m_keyword.size(); n++) kwunits[m_keyword[n]] = units[n]; - RstEntry rst_entry = std::get<1>(lodsmry_head); + RstEntry rst_entry = std::get<1>(ext_esmry_head); - m_rstep_v.push_back(std::get<4>(lodsmry_head)); - m_tstep_v.push_back(std::get<5>(lodsmry_head)); + m_rstep_v.push_back(std::get<4>(ext_esmry_head)); + m_tstep_v.push_back(std::get<5>(ext_esmry_head)); m_nTstep_v.push_back(m_tstep_v.back().size()); - auto lod_arr_size = sizeOnDiskBinary(m_nTstep_v.back(), Opm::EclIO::REAL, sizeOfReal); - - m_lod_arr_size.push_back(lod_arr_size); - m_tstep_range.push_back(std::make_tuple(0, m_tstep_v.back().size() - 1)); if ((loadBaseRunData) && (!std::get<0>(rst_entry).empty())) { @@ -162,23 +170,21 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) : updatePathAndRootName(path, rstRootN); - std::filesystem::path rstLodSmryFile = path / rstRootN; - rstLodSmryFile += ".ESMRY"; + std::filesystem::path rstESmryFile = path / rstRootN; + rstESmryFile += ".ESMRY"; - m_lodsmry_files.push_back(rstLodSmryFile); + m_esmry_files.push_back(rstESmryFile); - lod_offset = open_esmry(rstLodSmryFile, lodsmry_head); + if (!open_esmry(rstESmryFile, ext_esmry_head, rstep_offset)) + OPM_THROW( std::runtime_error, "when opening ESMRY file" + rstESmryFile.string() ); - m_lod_offset.push_back(lod_offset); + m_rstep_offset.push_back(rstep_offset); - m_rstep_v.push_back(std::get<4>(lodsmry_head)); - m_tstep_v.push_back(std::get<5>(lodsmry_head)); + m_rstep_v.push_back(std::get<4>(ext_esmry_head)); + m_tstep_v.push_back(std::get<5>(ext_esmry_head)); m_nTstep_v.push_back(m_tstep_v.back().size()); - lod_arr_size = sizeOnDiskBinary(m_nTstep_v.back(), Opm::EclIO::REAL, sizeOfReal); - m_lod_arr_size.push_back(lod_arr_size); - int cidx = 0; auto it = std::find_if(m_rstep_v[sim_ind].begin(), m_rstep_v[sim_ind].end(), @@ -195,14 +201,14 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) : m_tstep_range.push_back(std::make_tuple(0, ind)); key_index.clear(); - keyword = std::get<2>(lodsmry_head); + keyword = std::get<2>(ext_esmry_head); for (size_t n = 0; n < keyword.size(); n++) key_index[keyword[n]] = n; m_keyword_index.push_back(key_index); - rst_entry = std::get<1>(lodsmry_head); + rst_entry = std::get<1>(ext_esmry_head); restart = std::get<0>(rst_entry); rstNum = std::get<1>(rst_entry); } @@ -263,31 +269,46 @@ bool ExtESmry::all_steps_available() return true; } -uint64_t ExtESmry::open_esmry(std::filesystem::path& inputFileName, LodsmryHeadType& lodsmry_head) +bool ExtESmry::open_esmry(const std::filesystem::path& inputFileName, ExtSmryHeadType& ext_smry_head, uint64_t& rstep_offset) { std::fstream fileH; fileH.open(inputFileName, std::ios::in | std::ios::binary); if (!fileH) - throw std::runtime_error("Can not open file "); - + return false; std::string arrName; int64_t arr_size; Opm::EclIO::eclArrType arrType; int sizeOfElement; - Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + try { + Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } if ((arrName != "START ") or (arrType != Opm::EclIO::INTE)) - OPM_THROW(std::invalid_argument, "reading start, invalid lod file"); + OPM_THROW(std::invalid_argument, "reading start, invalid esmry file " + inputFileName.string() ); - auto start_vect = Opm::EclIO::readBinaryInteArray(fileH, arr_size); + std::vector start_vect; + try { + start_vect = Opm::EclIO::readBinaryInteArray(fileH, arr_size); + } catch (const std::runtime_error& error) + { + return false; + } auto startdat = make_date(start_vect); - Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + try { + Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } Opm::EclIO::RstEntry rst_entry = std::make_tuple("", 0); @@ -311,43 +332,85 @@ uint64_t ExtESmry::open_esmry(std::filesystem::path& inputFileName, LodsmryHeadT } if (arrName != "KEYCHECK") - OPM_THROW(std::invalid_argument, "!!reading keycheck, invalid lod file"); + OPM_THROW(std::invalid_argument, "reading keycheck, invalid esmry file " + inputFileName.string() ); std::vector keywords; - keywords = Opm::EclIO::readBinaryC0nnArray(fileH, arr_size, sizeOfElement); + try { + keywords = Opm::EclIO::readBinaryC0nnArray(fileH, arr_size, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } - Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + + try { + Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } if (arrName != "UNITS ") - OPM_THROW(std::invalid_argument, "reading UNITS, invalid lod file"); + OPM_THROW(std::invalid_argument, "reading UNITS, invalid esmry file " + inputFileName.string() ); - auto units = Opm::EclIO::readBinaryC0nnArray(fileH, arr_size, sizeOfElement); + std::vector units; + + try { + units = Opm::EclIO::readBinaryC0nnArray(fileH, arr_size, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } if (keywords.size() != units.size()) - throw std::runtime_error("invalid LODSMRY file, size of units not equal size of keywords"); + OPM_THROW( std::runtime_error, "invalid ESMRY file " + inputFileName.string() + ". Size of UNITS not equal size of KEYCHECK"); - Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + rstep_offset = static_cast(fileH.tellg()); + + try { + Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } if ((arrName != "RSTEP ") or (arrType != Opm::EclIO::INTE)) - OPM_THROW(std::invalid_argument, "reading RSTEP, invalid lod file"); + OPM_THROW(std::invalid_argument, "Reading RSTEP, invalid esmry file " + inputFileName.string() ); - auto rstep = Opm::EclIO::readBinaryInteArray(fileH, arr_size); + std::vector rstep; - Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + try { + rstep = Opm::EclIO::readBinaryInteArray(fileH, arr_size); + } catch (const std::runtime_error& error) + { + return false; + } + + try { + Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } if ((arrName != "TSTEP ") or (arrType != Opm::EclIO::INTE)) - OPM_THROW(std::invalid_argument, "reading TSTEP, invalid lod file"); + OPM_THROW(std::invalid_argument, "reading TSTEP, invalid esmry file " + inputFileName.string() ); - auto tstep = Opm::EclIO::readBinaryInteArray(fileH, arr_size); + std::vector tstep; - lodsmry_head = std::make_tuple(startdat, rst_entry, keywords, units, rstep, tstep); + try { + tstep = Opm::EclIO::readBinaryInteArray(fileH, arr_size); + } catch (const std::runtime_error& error) + { + return false; + } - uint64_t lodsmry_offset = static_cast(fileH.tellg()); + ext_smry_head = std::make_tuple(startdat, rst_entry, keywords, units, rstep, tstep); fileH.close(); - return lodsmry_offset; + return true; } @@ -363,6 +426,96 @@ void ExtESmry::updatePathAndRootName(std::filesystem::path& dir, std::filesystem } +bool ExtESmry::load_esmry(const std::vector& stringVect, const std::vector& keyIndexVect, + const std::vector& loadKeyIndex, int ind, int to_ind ) +{ + std::fstream fileH; + + fileH.open(m_esmry_files[ind], std::ios::in | std::ios::binary); + + if (!fileH) + return false; + + std::string arrName; + Opm::EclIO::eclArrType arrType; + int64_t num_tstep; + int sizeOfElement; + + // Read actual number of time steps on disk from RSTEP array before loading + // data. Notice that number of time steps can be different than what it was when + // the ESMRY file was opened. The simulation may have progressed if this is an + // ESMRY file from an active run + + fileH.seekg (m_rstep_offset[ind], fileH.beg); + + try { + Opm::EclIO::readBinaryHeader(fileH, arrName, num_tstep, arrType, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } + + auto smry_arr_size = sizeOnDiskBinary(num_tstep, Opm::EclIO::REAL, sizeOfReal); + + std::vector> smry_data; + smry_data.resize(loadKeyIndex.size(), {}); + + for (size_t n = 0 ; n < loadKeyIndex.size(); n++) { + + const auto& key = stringVect[loadKeyIndex[n]]; + + if ( m_keyword_index[ind].find(key) == m_keyword_index[ind].end() ) { + + smry_data[n].resize(to_ind + 1, 0.0 ); + + } else { + + int key_ind = m_keyword_index[ind].at(key); + + uint64_t pos = m_rstep_offset[ind] + smry_arr_size*static_cast(key_ind); + + // adding size of TSTEP and RSTEP INTE data + pos = pos + 2 * sizeOnDiskBinary(num_tstep, Opm::EclIO::INTE, sizeOfInte); + + pos = pos + static_cast(2 * 24); // adding size of binary headers (TSTEP and RSTEP) + pos = pos + static_cast(key_ind * 24); // adding size of binary headers + + fileH.seekg (pos, fileH.beg); + + int64_t size; + + try { + readBinaryHeader(fileH, arrName, size, arrType, sizeOfElement); + } catch (const std::runtime_error& error) + { + return false; + } + + arrName = Opm::EclIO::trimr(arrName); + + std::string checkName = "V" + std::to_string(key_ind); + + if (arrName != checkName) + return false; + + try { + smry_data[n] = readBinaryRealArray(fileH, size); + } catch (const std::runtime_error& error) + { + return false; + } + } + } + + fileH.close(); + + for (size_t n = 0 ; n < loadKeyIndex.size(); n++) + m_vectorData[keyIndexVect[n]].insert(m_vectorData[keyIndexVect[n]].end(), smry_data[n].begin(), smry_data[n].begin() + to_ind + 1); + + return true; +} + + void ExtESmry::loadData(const std::vector& stringVect) { auto start = std::chrono::system_clock::now(); @@ -385,59 +538,27 @@ void ExtESmry::loadData(const std::vector& stringVect) ++keyCounter; } - std::fstream fileH; - int ind = static_cast(m_tstep_range.size()) - 1 ; while (ind > -1) { int to_ind = std::get<1>(m_tstep_range[ind]); - fileH.open(m_lodsmry_files[ind], std::ios::in | std::ios::binary); + bool res = load_esmry(stringVect, keyIndexVect, loadKeyIndex, ind, to_ind ); - if (!fileH) - throw std::runtime_error("Can not open file lodFile"); + int n_attempts = 1; - for (size_t n = 0 ; n < loadKeyIndex.size(); n++) { - - //std::string key = loadKeyNames[n]; - const auto& key = stringVect[loadKeyIndex[n]]; - - std::string arrName; - Opm::EclIO::eclArrType arrType; - - if ( m_keyword_index[ind].find(key) == m_keyword_index[ind].end() ) { - - for (int m = 0; m < to_ind + 1; m++) - m_vectorData[keyIndexVect[n]].push_back(0.0); - - } else { - - int key_ind = m_keyword_index[ind].at(key); - - uint64_t pos = m_lod_offset[ind] + m_lod_arr_size[ind]*static_cast(key_ind); - pos = pos + static_cast(key_ind * 24); // adding size of binary headers - - fileH.seekg (pos, fileH.beg); - - int64_t size; - int sizeOfElement; - readBinaryHeader(fileH, arrName, size, arrType, sizeOfElement); - - arrName = Opm::EclIO::trimr(arrName); - - std::string checkName = "V" + std::to_string(key_ind); - - if (arrName != checkName) - OPM_THROW(std::invalid_argument, "lodsmry, wrong header expecting " + checkName + " found " + arrName); - - auto smry_data = readBinaryRealArray(fileH, size); - - m_vectorData[keyIndexVect[n]].insert(m_vectorData[keyIndexVect[n]].end(), smry_data.begin(), smry_data.begin() + to_ind + 1); - } + while ((!res) && (n_attempts < 10)){ + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + res = load_esmry(stringVect, keyIndexVect, loadKeyIndex, ind, to_ind ); + n_attempts ++; + } + + if (n_attempts == 10){ + std::string emsry_file_name = m_esmry_files[ind].string(); + OPM_THROW( std::runtime_error, "when loading data from ESMRY file" + emsry_file_name ); } - fileH.close(); ind--; }