ExtESmry - support loading when number of time steps have increased (on disk)
The ESMRY file can be updated with more time step data in the time span between opening an ESMRY file and loading actual summary data from the file. This can happen if the ESMRY is generated by an active run. Length of summary vectors extracted will be equal to number of time steps on disk when the file was first opened (with the constructor).
This commit is contained in:
Torbjørn Skille
parent
ab81083e02
commit
1433bc7a55
@ -37,7 +37,7 @@ using TimeStepEntry = std::tuple<int, int, uint64_t>;
|
||||
using RstEntry = std::tuple<std::string, int>;
|
||||
|
||||
// start, rstart + rstnum, keycheck, units, rstep, tstep
|
||||
using LodsmryHeadType = std::tuple<time_point, RstEntry, std::vector<std::string>, std::vector<std::string>,
|
||||
using ExtSmryHeadType = std::tuple<time_point, RstEntry, std::vector<std::string>, std::vector<std::string>,
|
||||
std::vector<int>, std::vector<int>>;
|
||||
|
||||
class ExtESmry
|
||||
@ -72,7 +72,7 @@ public:
|
||||
|
||||
private:
|
||||
std::filesystem::path m_inputFileName;
|
||||
std::vector<std::filesystem::path> m_lodsmry_files;
|
||||
std::vector<std::filesystem::path> m_esmry_files;
|
||||
|
||||
bool m_loadBaseRun;
|
||||
std::vector<std::map<std::string, int>> m_keyword_index;
|
||||
@ -91,15 +91,17 @@ private:
|
||||
size_t m_nTstep;
|
||||
std::vector<int> m_seqIndex;
|
||||
|
||||
std::vector<uint64_t> m_lod_offset;
|
||||
std::vector<uint64_t> m_lod_arr_size;
|
||||
std::vector<uint64_t> m_rstep_offset;
|
||||
|
||||
time_point m_startdat;
|
||||
|
||||
double m_io_opening;
|
||||
double m_io_loading;
|
||||
|
||||
uint64_t open_esmry(std::filesystem::path& inputFileName, LodsmryHeadType& lodsmry_head);
|
||||
bool open_esmry(const std::filesystem::path& inputFileName, ExtSmryHeadType& ext_smry_head, uint64_t& rstep_offset);
|
||||
|
||||
bool load_esmry(const std::vector<std::string>& stringVect, const std::vector<int>& keyIndexVect,
|
||||
const std::vector<int>& loadKeyIndex, int ind, int to_ind );
|
||||
|
||||
void updatePathAndRootName(std::filesystem::path& dir, std::filesystem::path& rootN);
|
||||
};
|
||||
|
||||
@ -38,6 +38,7 @@
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
|
||||
|
||||
namespace {
|
||||
@ -104,7 +105,7 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) :
|
||||
if (m_inputFileName.extension()!=".ESMRY")
|
||||
throw std::invalid_argument("Input file should have extension .ESMRY");
|
||||
|
||||
m_lodsmry_files.push_back(m_inputFileName);
|
||||
m_esmry_files.push_back(m_inputFileName);
|
||||
|
||||
std::filesystem::path rootName = m_inputFileName.parent_path() / m_inputFileName.stem();
|
||||
std::filesystem::path path = std::filesystem::current_path();
|
||||
@ -113,18 +114,29 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) :
|
||||
|
||||
updatePathAndRootName(path, rootName);
|
||||
|
||||
LodsmryHeadType lodsmry_head;
|
||||
ExtSmryHeadType ext_esmry_head;
|
||||
|
||||
auto lod_offset = open_esmry(m_inputFileName, lodsmry_head);
|
||||
uint64_t rstep_offset;
|
||||
|
||||
m_startdat = std::get<0>(lodsmry_head);
|
||||
bool res = open_esmry(m_inputFileName, ext_esmry_head, rstep_offset);
|
||||
int n_attempts = 1;
|
||||
|
||||
m_lod_offset.push_back(lod_offset);
|
||||
while ((!res) && (n_attempts < 10)){
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
res = open_esmry(m_inputFileName, ext_esmry_head, rstep_offset);
|
||||
n_attempts ++;
|
||||
}
|
||||
|
||||
if (n_attempts == 10)
|
||||
OPM_THROW( std::runtime_error, "when opening ESMRY file " + filename );
|
||||
|
||||
m_startdat = std::get<0>(ext_esmry_head);
|
||||
m_rstep_offset.push_back(rstep_offset);
|
||||
|
||||
std::map<std::string, int> key_index;
|
||||
|
||||
auto keyword = std::get<2>(lodsmry_head);
|
||||
auto units = std::get<3>(lodsmry_head);
|
||||
auto keyword = std::get<2>(ext_esmry_head);
|
||||
auto units = std::get<3>(ext_esmry_head);
|
||||
|
||||
for (size_t n = 0; n < keyword.size(); n++){
|
||||
key_index[keyword[n]] = n;
|
||||
@ -136,17 +148,13 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) :
|
||||
for (size_t n = 0; n < m_keyword.size(); n++)
|
||||
kwunits[m_keyword[n]] = units[n];
|
||||
|
||||
RstEntry rst_entry = std::get<1>(lodsmry_head);
|
||||
RstEntry rst_entry = std::get<1>(ext_esmry_head);
|
||||
|
||||
m_rstep_v.push_back(std::get<4>(lodsmry_head));
|
||||
m_tstep_v.push_back(std::get<5>(lodsmry_head));
|
||||
m_rstep_v.push_back(std::get<4>(ext_esmry_head));
|
||||
m_tstep_v.push_back(std::get<5>(ext_esmry_head));
|
||||
|
||||
m_nTstep_v.push_back(m_tstep_v.back().size());
|
||||
|
||||
auto lod_arr_size = sizeOnDiskBinary(m_nTstep_v.back(), Opm::EclIO::REAL, sizeOfReal);
|
||||
|
||||
m_lod_arr_size.push_back(lod_arr_size);
|
||||
|
||||
m_tstep_range.push_back(std::make_tuple(0, m_tstep_v.back().size() - 1));
|
||||
|
||||
if ((loadBaseRunData) && (!std::get<0>(rst_entry).empty())) {
|
||||
@ -162,23 +170,21 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) :
|
||||
|
||||
updatePathAndRootName(path, rstRootN);
|
||||
|
||||
std::filesystem::path rstLodSmryFile = path / rstRootN;
|
||||
rstLodSmryFile += ".ESMRY";
|
||||
std::filesystem::path rstESmryFile = path / rstRootN;
|
||||
rstESmryFile += ".ESMRY";
|
||||
|
||||
m_lodsmry_files.push_back(rstLodSmryFile);
|
||||
m_esmry_files.push_back(rstESmryFile);
|
||||
|
||||
lod_offset = open_esmry(rstLodSmryFile, lodsmry_head);
|
||||
if (!open_esmry(rstESmryFile, ext_esmry_head, rstep_offset))
|
||||
OPM_THROW( std::runtime_error, "when opening ESMRY file" + rstESmryFile.string() );
|
||||
|
||||
m_lod_offset.push_back(lod_offset);
|
||||
m_rstep_offset.push_back(rstep_offset);
|
||||
|
||||
m_rstep_v.push_back(std::get<4>(lodsmry_head));
|
||||
m_tstep_v.push_back(std::get<5>(lodsmry_head));
|
||||
m_rstep_v.push_back(std::get<4>(ext_esmry_head));
|
||||
m_tstep_v.push_back(std::get<5>(ext_esmry_head));
|
||||
|
||||
m_nTstep_v.push_back(m_tstep_v.back().size());
|
||||
|
||||
lod_arr_size = sizeOnDiskBinary(m_nTstep_v.back(), Opm::EclIO::REAL, sizeOfReal);
|
||||
m_lod_arr_size.push_back(lod_arr_size);
|
||||
|
||||
int cidx = 0;
|
||||
|
||||
auto it = std::find_if(m_rstep_v[sim_ind].begin(), m_rstep_v[sim_ind].end(),
|
||||
@ -195,14 +201,14 @@ ExtESmry::ExtESmry(const std::string &filename, bool loadBaseRunData) :
|
||||
m_tstep_range.push_back(std::make_tuple(0, ind));
|
||||
|
||||
key_index.clear();
|
||||
keyword = std::get<2>(lodsmry_head);
|
||||
keyword = std::get<2>(ext_esmry_head);
|
||||
|
||||
for (size_t n = 0; n < keyword.size(); n++)
|
||||
key_index[keyword[n]] = n;
|
||||
|
||||
m_keyword_index.push_back(key_index);
|
||||
|
||||
rst_entry = std::get<1>(lodsmry_head);
|
||||
rst_entry = std::get<1>(ext_esmry_head);
|
||||
restart = std::get<0>(rst_entry);
|
||||
rstNum = std::get<1>(rst_entry);
|
||||
}
|
||||
@ -263,31 +269,46 @@ bool ExtESmry::all_steps_available()
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t ExtESmry::open_esmry(std::filesystem::path& inputFileName, LodsmryHeadType& lodsmry_head)
|
||||
bool ExtESmry::open_esmry(const std::filesystem::path& inputFileName, ExtSmryHeadType& ext_smry_head, uint64_t& rstep_offset)
|
||||
{
|
||||
std::fstream fileH;
|
||||
|
||||
fileH.open(inputFileName, std::ios::in | std::ios::binary);
|
||||
|
||||
if (!fileH)
|
||||
throw std::runtime_error("Can not open file ");
|
||||
|
||||
return false;
|
||||
|
||||
std::string arrName;
|
||||
int64_t arr_size;
|
||||
Opm::EclIO::eclArrType arrType;
|
||||
int sizeOfElement;
|
||||
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
try {
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((arrName != "START ") or (arrType != Opm::EclIO::INTE))
|
||||
OPM_THROW(std::invalid_argument, "reading start, invalid lod file");
|
||||
OPM_THROW(std::invalid_argument, "reading start, invalid esmry file " + inputFileName.string() );
|
||||
|
||||
auto start_vect = Opm::EclIO::readBinaryInteArray(fileH, arr_size);
|
||||
std::vector<int> start_vect;
|
||||
try {
|
||||
start_vect = Opm::EclIO::readBinaryInteArray(fileH, arr_size);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
auto startdat = make_date(start_vect);
|
||||
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
try {
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Opm::EclIO::RstEntry rst_entry = std::make_tuple("", 0);
|
||||
|
||||
@ -311,43 +332,85 @@ uint64_t ExtESmry::open_esmry(std::filesystem::path& inputFileName, LodsmryHeadT
|
||||
}
|
||||
|
||||
if (arrName != "KEYCHECK")
|
||||
OPM_THROW(std::invalid_argument, "!!reading keycheck, invalid lod file");
|
||||
OPM_THROW(std::invalid_argument, "reading keycheck, invalid esmry file " + inputFileName.string() );
|
||||
|
||||
std::vector<std::string> keywords;
|
||||
|
||||
keywords = Opm::EclIO::readBinaryC0nnArray(fileH, arr_size, sizeOfElement);
|
||||
try {
|
||||
keywords = Opm::EclIO::readBinaryC0nnArray(fileH, arr_size, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
|
||||
try {
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (arrName != "UNITS ")
|
||||
OPM_THROW(std::invalid_argument, "reading UNITS, invalid lod file");
|
||||
OPM_THROW(std::invalid_argument, "reading UNITS, invalid esmry file " + inputFileName.string() );
|
||||
|
||||
auto units = Opm::EclIO::readBinaryC0nnArray(fileH, arr_size, sizeOfElement);
|
||||
std::vector<std::string> units;
|
||||
|
||||
try {
|
||||
units = Opm::EclIO::readBinaryC0nnArray(fileH, arr_size, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (keywords.size() != units.size())
|
||||
throw std::runtime_error("invalid LODSMRY file, size of units not equal size of keywords");
|
||||
OPM_THROW( std::runtime_error, "invalid ESMRY file " + inputFileName.string() + ". Size of UNITS not equal size of KEYCHECK");
|
||||
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
rstep_offset = static_cast<uint64_t>(fileH.tellg());
|
||||
|
||||
try {
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((arrName != "RSTEP ") or (arrType != Opm::EclIO::INTE))
|
||||
OPM_THROW(std::invalid_argument, "reading RSTEP, invalid lod file");
|
||||
OPM_THROW(std::invalid_argument, "Reading RSTEP, invalid esmry file " + inputFileName.string() );
|
||||
|
||||
auto rstep = Opm::EclIO::readBinaryInteArray(fileH, arr_size);
|
||||
std::vector<int> rstep;
|
||||
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
try {
|
||||
rstep = Opm::EclIO::readBinaryInteArray(fileH, arr_size);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, arr_size, arrType, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((arrName != "TSTEP ") or (arrType != Opm::EclIO::INTE))
|
||||
OPM_THROW(std::invalid_argument, "reading TSTEP, invalid lod file");
|
||||
OPM_THROW(std::invalid_argument, "reading TSTEP, invalid esmry file " + inputFileName.string() );
|
||||
|
||||
auto tstep = Opm::EclIO::readBinaryInteArray(fileH, arr_size);
|
||||
std::vector<int> tstep;
|
||||
|
||||
lodsmry_head = std::make_tuple(startdat, rst_entry, keywords, units, rstep, tstep);
|
||||
try {
|
||||
tstep = Opm::EclIO::readBinaryInteArray(fileH, arr_size);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t lodsmry_offset = static_cast<uint64_t>(fileH.tellg());
|
||||
ext_smry_head = std::make_tuple(startdat, rst_entry, keywords, units, rstep, tstep);
|
||||
|
||||
fileH.close();
|
||||
|
||||
return lodsmry_offset;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -363,6 +426,96 @@ void ExtESmry::updatePathAndRootName(std::filesystem::path& dir, std::filesystem
|
||||
}
|
||||
|
||||
|
||||
bool ExtESmry::load_esmry(const std::vector<std::string>& stringVect, const std::vector<int>& keyIndexVect,
|
||||
const std::vector<int>& loadKeyIndex, int ind, int to_ind )
|
||||
{
|
||||
std::fstream fileH;
|
||||
|
||||
fileH.open(m_esmry_files[ind], std::ios::in | std::ios::binary);
|
||||
|
||||
if (!fileH)
|
||||
return false;
|
||||
|
||||
std::string arrName;
|
||||
Opm::EclIO::eclArrType arrType;
|
||||
int64_t num_tstep;
|
||||
int sizeOfElement;
|
||||
|
||||
// Read actual number of time steps on disk from RSTEP array before loading
|
||||
// data. Notice that number of time steps can be different than what it was when
|
||||
// the ESMRY file was opened. The simulation may have progressed if this is an
|
||||
// ESMRY file from an active run
|
||||
|
||||
fileH.seekg (m_rstep_offset[ind], fileH.beg);
|
||||
|
||||
try {
|
||||
Opm::EclIO::readBinaryHeader(fileH, arrName, num_tstep, arrType, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
auto smry_arr_size = sizeOnDiskBinary(num_tstep, Opm::EclIO::REAL, sizeOfReal);
|
||||
|
||||
std::vector<std::vector<float>> smry_data;
|
||||
smry_data.resize(loadKeyIndex.size(), {});
|
||||
|
||||
for (size_t n = 0 ; n < loadKeyIndex.size(); n++) {
|
||||
|
||||
const auto& key = stringVect[loadKeyIndex[n]];
|
||||
|
||||
if ( m_keyword_index[ind].find(key) == m_keyword_index[ind].end() ) {
|
||||
|
||||
smry_data[n].resize(to_ind + 1, 0.0 );
|
||||
|
||||
} else {
|
||||
|
||||
int key_ind = m_keyword_index[ind].at(key);
|
||||
|
||||
uint64_t pos = m_rstep_offset[ind] + smry_arr_size*static_cast<uint64_t>(key_ind);
|
||||
|
||||
// adding size of TSTEP and RSTEP INTE data
|
||||
pos = pos + 2 * sizeOnDiskBinary(num_tstep, Opm::EclIO::INTE, sizeOfInte);
|
||||
|
||||
pos = pos + static_cast<uint64_t>(2 * 24); // adding size of binary headers (TSTEP and RSTEP)
|
||||
pos = pos + static_cast<uint64_t>(key_ind * 24); // adding size of binary headers
|
||||
|
||||
fileH.seekg (pos, fileH.beg);
|
||||
|
||||
int64_t size;
|
||||
|
||||
try {
|
||||
readBinaryHeader(fileH, arrName, size, arrType, sizeOfElement);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
arrName = Opm::EclIO::trimr(arrName);
|
||||
|
||||
std::string checkName = "V" + std::to_string(key_ind);
|
||||
|
||||
if (arrName != checkName)
|
||||
return false;
|
||||
|
||||
try {
|
||||
smry_data[n] = readBinaryRealArray(fileH, size);
|
||||
} catch (const std::runtime_error& error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fileH.close();
|
||||
|
||||
for (size_t n = 0 ; n < loadKeyIndex.size(); n++)
|
||||
m_vectorData[keyIndexVect[n]].insert(m_vectorData[keyIndexVect[n]].end(), smry_data[n].begin(), smry_data[n].begin() + to_ind + 1);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void ExtESmry::loadData(const std::vector<std::string>& stringVect)
|
||||
{
|
||||
auto start = std::chrono::system_clock::now();
|
||||
@ -385,59 +538,27 @@ void ExtESmry::loadData(const std::vector<std::string>& stringVect)
|
||||
++keyCounter;
|
||||
}
|
||||
|
||||
std::fstream fileH;
|
||||
|
||||
int ind = static_cast<int>(m_tstep_range.size()) - 1 ;
|
||||
|
||||
while (ind > -1) {
|
||||
|
||||
int to_ind = std::get<1>(m_tstep_range[ind]);
|
||||
|
||||
fileH.open(m_lodsmry_files[ind], std::ios::in | std::ios::binary);
|
||||
bool res = load_esmry(stringVect, keyIndexVect, loadKeyIndex, ind, to_ind );
|
||||
|
||||
if (!fileH)
|
||||
throw std::runtime_error("Can not open file lodFile");
|
||||
int n_attempts = 1;
|
||||
|
||||
for (size_t n = 0 ; n < loadKeyIndex.size(); n++) {
|
||||
|
||||
//std::string key = loadKeyNames[n];
|
||||
const auto& key = stringVect[loadKeyIndex[n]];
|
||||
|
||||
std::string arrName;
|
||||
Opm::EclIO::eclArrType arrType;
|
||||
|
||||
if ( m_keyword_index[ind].find(key) == m_keyword_index[ind].end() ) {
|
||||
|
||||
for (int m = 0; m < to_ind + 1; m++)
|
||||
m_vectorData[keyIndexVect[n]].push_back(0.0);
|
||||
|
||||
} else {
|
||||
|
||||
int key_ind = m_keyword_index[ind].at(key);
|
||||
|
||||
uint64_t pos = m_lod_offset[ind] + m_lod_arr_size[ind]*static_cast<uint64_t>(key_ind);
|
||||
pos = pos + static_cast<uint64_t>(key_ind * 24); // adding size of binary headers
|
||||
|
||||
fileH.seekg (pos, fileH.beg);
|
||||
|
||||
int64_t size;
|
||||
int sizeOfElement;
|
||||
readBinaryHeader(fileH, arrName, size, arrType, sizeOfElement);
|
||||
|
||||
arrName = Opm::EclIO::trimr(arrName);
|
||||
|
||||
std::string checkName = "V" + std::to_string(key_ind);
|
||||
|
||||
if (arrName != checkName)
|
||||
OPM_THROW(std::invalid_argument, "lodsmry, wrong header expecting " + checkName + " found " + arrName);
|
||||
|
||||
auto smry_data = readBinaryRealArray(fileH, size);
|
||||
|
||||
m_vectorData[keyIndexVect[n]].insert(m_vectorData[keyIndexVect[n]].end(), smry_data.begin(), smry_data.begin() + to_ind + 1);
|
||||
}
|
||||
while ((!res) && (n_attempts < 10)){
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
res = load_esmry(stringVect, keyIndexVect, loadKeyIndex, ind, to_ind );
|
||||
n_attempts ++;
|
||||
}
|
||||
|
||||
if (n_attempts == 10){
|
||||
std::string emsry_file_name = m_esmry_files[ind].string();
|
||||
OPM_THROW( std::runtime_error, "when loading data from ESMRY file" + emsry_file_name );
|
||||
}
|
||||
|
||||
fileH.close();
|
||||
ind--;
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user