#8143 Add EclipseTextFileReader with unit test

unit

ApplicationLibCode/FileInterface/CMakeLists_files.cmake

@@ -67,8 +67,7 @@ set(SOURCE_GROUP_HEADER_FILES
     ${CMAKE_CURRENT_LIST_DIR}/RifSummaryReaderMultipleFiles.h
     ${CMAKE_CURRENT_LIST_DIR}/RifEclEclipseSummary.h
     ${CMAKE_CURRENT_LIST_DIR}/RifWellIAFileWriter.h
-    # HDF5 file reader is directly included in ResInsight main CmakeList.txt
-    # ${CMAKE_CURRENT_LIST_DIR}/RifHdf5Reader.h
+    ${CMAKE_CURRENT_LIST_DIR}/RifEclipseTextFileReader.h
 )
 
 set(SOURCE_GROUP_SOURCE_FILES
@@ -138,8 +137,7 @@ set(SOURCE_GROUP_SOURCE_FILES
     ${CMAKE_CURRENT_LIST_DIR}/RifSummaryReaderMultipleFiles.cpp
     ${CMAKE_CURRENT_LIST_DIR}/RifEclEclipseSummary.cpp
     ${CMAKE_CURRENT_LIST_DIR}/RifWellIAFileWriter.cpp
-    # HDF5 file reader is directly included in ResInsight main CmakeList.txt
-    # ${CMAKE_CURRENT_LIST_DIR}/RifHdf5Reader.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/RifEclipseTextFileReader.cpp
 )
 
 list(APPEND CODE_HEADER_FILES ${SOURCE_GROUP_HEADER_FILES})

ApplicationLibCode/FileInterface/RifEclipseTextFileReader.cpp

@@ -0,0 +1,249 @@
+/////////////////////////////////////////////////////////////////////////////////
+//
+//  Copyright (C) 2021     Equinor ASA
+//
+//  ResInsight is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
+//  WARRANTY; without even the implied warranty of MERCHANTABILITY or
+//  FITNESS FOR A PARTICULAR PURPOSE.
+//
+//  See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
+//  for more details.
+//
+/////////////////////////////////////////////////////////////////////////////////
+
+#include "RifEclipseTextFileReader.h"
+
+// Utility class for fast conversion from string to float
+#include "fast_float/include/fast_float/fast_float.h"
+
+// Utility class memory mapping of files
+#include "mio/mio.hpp"
+
+#include "RiaPreferencesSystem.h"
+
+#include <fstream>
+#include <iosfwd>
+#include <iostream>
+#include <sstream>
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::vector<RifEclipseKeywordContent> RifEclipseTextFileReader::readKeywordAndValues( const std::string& filename )
+{
+    if ( RiaPreferencesSystem::current()->eclipseTextFileReaderMode() ==
+         RiaPreferencesSystem::EclipseTextFileReaderMode::MEMORY_MAPPED_FILE )
+    {
+        return readKeywordAndValuesMemoryMappedFile( filename );
+    }
+
+    return readKeywordAndValuesFile( filename );
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::vector<RifEclipseKeywordContent> RifEclipseTextFileReader::readKeywordAndValuesFile( const std::string& filename )
+{
+    std::string stringData;
+
+    std::ifstream inFile;
+    inFile.open( filename );
+
+    std::stringstream strStream;
+    strStream << inFile.rdbuf();
+
+    stringData = strStream.str();
+
+    return parseStringData( stringData );
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::vector<RifEclipseKeywordContent>
+    RifEclipseTextFileReader::readKeywordAndValuesMemoryMappedFile( const std::string& filename )
+{
+    mio::mmap_source mmap( filename );
+
+    std::error_code  error;
+    mio::mmap_sink   rw_mmap    = mio::make_mmap_sink( filename, 0, mio::map_entire_file, error );
+    std::string_view stringData = rw_mmap.data();
+
+    return parseStringData( stringData );
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::pair<std::string, std::vector<float>>
+    RifEclipseTextFileReader::readKeywordAndValues( const std::string_view& stringData, const size_t offset, size_t& bytesRead )
+{
+    std::vector<float> values;
+
+    const auto commentChar = '-';
+
+    std::string keywordName;
+    std::string line;
+    bool        isEndTokenKeywordRead = false;
+
+    bytesRead   = 0;
+    float value = 0.0f;
+
+    bool isFaultKeyword = false;
+
+    while ( !isEndTokenKeywordRead && ( offset + bytesRead < stringData.size() ) )
+    {
+        size_t bytesReadLine = 0;
+        line                 = readLine( stringData, offset + bytesRead, bytesReadLine );
+        bytesRead += bytesReadLine;
+
+        if ( isFaultKeyword )
+        {
+            // Read data until the FAULTS section is closed with a single / on one line
+            const auto& trimmed = ltrim( line );
+            if ( !trimmed.empty() && trimmed[0] == '/' )
+            {
+                return std::make_pair( keywordName, values );
+            }
+            continue;
+        }
+
+        // Check for '--', used to define start of comments
+        if ( line.size() > 2 && line[0] == commentChar && line[1] == commentChar ) continue;
+
+        if ( keywordName.empty() )
+        {
+            std::string candidate = trim( line );
+            if ( !candidate.empty() )
+            {
+                keywordName = candidate;
+                if ( keywordName == "FAULTS" ) isFaultKeyword = true;
+            }
+            continue;
+        }
+
+        if ( !isEndTokenKeywordRead )
+        {
+            size_t start = 0;
+            size_t end   = 0;
+
+            while ( ( start = line.find_first_not_of( RifEclipseTextFileReader::m_whiteSpace, end ) ) != std::string::npos )
+            {
+                end = line.find_first_of( RifEclipseTextFileReader::m_whiteSpace, start );
+
+                auto resultObject = fast_float::from_chars( line.data() + start, line.data() + end, value );
+                if ( resultObject.ec == std::errc() )
+                {
+                    values.emplace_back( value );
+                }
+            }
+        }
+
+        // End of keyword is defined by '/'
+        if ( line.find_first_of( '/' ) != std::string::npos )
+        {
+            isEndTokenKeywordRead = true;
+            continue;
+        }
+    }
+
+    return std::make_pair( keywordName, values );
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::vector<RifEclipseKeywordContent> RifEclipseTextFileReader::parseStringData( const std::string_view& stringData )
+{
+    size_t offset    = 0;
+    size_t bytesRead = 0;
+
+    std::vector<RifEclipseKeywordContent> dataObjects;
+
+    while ( offset < stringData.size() )
+    {
+        auto [keyword, values] = RifEclipseTextFileReader::readKeywordAndValues( stringData, offset, bytesRead );
+
+        if ( !keyword.empty() )
+        {
+            RifEclipseKeywordContent content;
+            content.keyword = keyword;
+            content.offset  = offset;
+            if ( values.empty() )
+            {
+                content.content = stringData.substr( offset, bytesRead );
+            }
+            else
+            {
+                content.values = values;
+            }
+
+            dataObjects.emplace_back( content );
+        }
+
+        offset += bytesRead;
+    }
+
+    return dataObjects;
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::string_view RifEclipseTextFileReader::readLine( const std::string_view& source, const size_t offset, size_t& bytesRead )
+{
+    if ( offset >= source.size() ) return {};
+
+    auto start = source.find_first_not_of( RifEclipseTextFileReader::m_whiteSpace, offset );
+    if ( start == std::string::npos )
+    {
+        bytesRead = source.size() - offset;
+        return source.substr( offset, bytesRead );
+    }
+
+    auto end = source.find_first_of( '\n', start );
+    if ( end != std::string::npos )
+    {
+        // Add 1 to skip the \n we have found
+        bytesRead = end - offset + 1;
+        return source.substr( start, end - start );
+    }
+
+    // No line shift found, reached end of string data
+    end = source.size();
+
+    bytesRead = end - offset;
+    return source.substr( start, bytesRead );
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::string& RifEclipseTextFileReader::rtrim( std::string& s, const char* t /*= ws */ )
+{
+    s.erase( s.find_last_not_of( t ) + 1 );
+    return s;
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::string& RifEclipseTextFileReader::ltrim( std::string& s, const char* t /*= ws */ )
+{
+    s.erase( 0, s.find_first_not_of( t ) );
+    return s;
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+std::string& RifEclipseTextFileReader::trim( std::string& s, const char* t /*= ws */ )
+{
+    return ltrim( rtrim( s, t ), t );
+}

ApplicationLibCode/FileInterface/RifEclipseTextFileReader.h

@@ -0,0 +1,71 @@
+/////////////////////////////////////////////////////////////////////////////////
+//
+//  Copyright (C) 2021     Equinor ASA
+//
+//  ResInsight is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
+//  WARRANTY; without even the implied warranty of MERCHANTABILITY or
+//  FITNESS FOR A PARTICULAR PURPOSE.
+//
+//  See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
+//  for more details.
+//
+/////////////////////////////////////////////////////////////////////////////////
+
+#pragma once
+
+#include "RifReaderInterface.h"
+
+class RifEclipseKeywordContent
+{
+public:
+    std::string        keyword;
+    std::string        content;
+    std::vector<float> values;
+    size_t             offset;
+};
+
+//==================================================================================================
+//
+// File interface for Eclipse output files
+//
+//==================================================================================================
+class RifEclipseTextFileReader
+{
+public:
+    // Settings in Preferences will be used to forward to either file-direct or memory mapped file reader
+    static std::vector<RifEclipseKeywordContent> readKeywordAndValues( const std::string& filename );
+
+    // Read data directly from file
+    static std::vector<RifEclipseKeywordContent> readKeywordAndValuesFile( const std::string& filename );
+
+    // Read data using memory mapped file
+    static std::vector<RifEclipseKeywordContent> readKeywordAndValuesMemoryMappedFile( const std::string& filename );
+
+    // Data import function, public to be able to use from unit test
+    static std::pair<std::string, std::vector<float>>
+        readKeywordAndValues( const std::string_view& stringData, const size_t startOffset, size_t& bytesRead );
+
+private:
+    static constexpr const char* m_whiteSpace = " \t\n\r\f\v";
+
+    // TODO: Make private or move to separate file, now public to be able to test code
+public:
+    static std::string_view readLine( const std::string_view& source, const size_t offset, size_t& bytesRead );
+
+    // trim from end of string (right)
+    static std::string& rtrim( std::string& s, const char* t = m_whiteSpace );
+
+    // trim from beginning of string (left)
+    static std::string& ltrim( std::string& s, const char* t = m_whiteSpace );
+
+    // trim from both ends of string (right then left)
+    static std::string& trim( std::string& s, const char* t = m_whiteSpace );
+
+    // Parse string data for Eclipse keywords
+    static std::vector<RifEclipseKeywordContent> parseStringData( const std::string_view& stringData );
+};

ApplicationLibCode/UnitTests/CMakeLists_files.cmake

@@ -80,6 +80,7 @@ set(SOURCE_GROUP_SOURCE_FILES
     ${CMAKE_CURRENT_LIST_DIR}/RigSurfaceStatisticsCalculator-Test.cpp
     ${CMAKE_CURRENT_LIST_DIR}/StructGridInterface-Test.cpp
     ${CMAKE_CURRENT_LIST_DIR}/opm-summary-Test.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/RifEclipseTextFileReader-Test.cpp
 )
 
 if(RESINSIGHT_ENABLE_GRPC)

ApplicationLibCode/UnitTests/RifEclipseTextFileReader-Test.cpp

@@ -0,0 +1,156 @@
+/////////////////////////////////////////////////////////////////////////////////
+//
+//  Copyright (C) 2020- Equinor ASA
+//
+//  ResInsight is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
+//  WARRANTY; without even the implied warranty of MERCHANTABILITY or
+//  FITNESS FOR A PARTICULAR PURPOSE.
+//
+//  See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
+//  for more details.
+//
+/////////////////////////////////////////////////////////////////////////////////
+
+#include "gtest/gtest.h"
+
+#include "RiaTestDataDirectory.h"
+#include "RifEclipseTextFileReader.h"
+
+#include <QString>
+#include <chrono>
+#include <fstream>
+#include <iosfwd>
+#include <iostream>
+#include <sstream>
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+TEST( RifEclipseTextFileReader, DISABLED_ReadKeywordsAndValuesPerformanceTest )
+{
+    // std::string filename       = "d:/scratchMsj/2021-10-13-norne-1M-single-grdecl/GRID.GRDECL";
+    // std::string filename = "e:/project/2021-10-13-norne-1M-single-grdecl/GRID.GRDECL";
+    std::string filename       = "c:/temp/GRID.GRDECL";
+    size_t      iterationCount = 10;
+
+    {
+        auto aggregatedStart = std::chrono::high_resolution_clock::now();
+
+        for ( size_t i = 0; i < iterationCount; i++ )
+        {
+            auto start = std::chrono::high_resolution_clock::now();
+
+            auto objects = RifEclipseTextFileReader::readKeywordAndValuesMemoryMappedFile( filename );
+
+            auto                          end  = std::chrono::high_resolution_clock::now();
+            std::chrono::duration<double> diff = end - start;
+            std::cout << "RifEclipseTextFileReader MM : " << std::setw( 9 ) << diff.count() << " s\n";
+        }
+
+        auto                          end  = std::chrono::high_resolution_clock::now();
+        std::chrono::duration<double> diff = end - aggregatedStart;
+        std::cout << "RifEclipseTextFileReader MM [" << iterationCount << " runs] : " << std::setw( 9 ) << diff.count()
+                  << " s\n";
+    }
+
+    {
+        auto aggregatedStart = std::chrono::high_resolution_clock::now();
+
+        for ( size_t i = 0; i < iterationCount; i++ )
+        {
+            auto start = std::chrono::high_resolution_clock::now();
+
+            auto objects = RifEclipseTextFileReader::readKeywordAndValuesFile( filename );
+
+            auto                          end  = std::chrono::high_resolution_clock::now();
+            std::chrono::duration<double> diff = end - start;
+            std::cout << "RifEclipseTextFileReader File : " << std::setw( 9 ) << diff.count() << " s\n";
+        }
+
+        auto                          end  = std::chrono::high_resolution_clock::now();
+        std::chrono::duration<double> diff = end - aggregatedStart;
+        std::cout << "RifEclipseTextFileReader File [" << iterationCount << " runs] : " << std::setw( 9 )
+                  << diff.count() << " s\n";
+    }
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+TEST( RifEclipseTextFileReader, ReadKeywordsAndValuesFromFile )
+{
+    QString qtFileName = QString( "%1/RifEclipseTextFileParser/GRID.GRDECL" ).arg( TEST_DATA_DIR );
+    //    "e:/gitroot-ceesol/ResInsight-regression-test/ModelData/TestCase_Ascii_no_map_axis/geocell.grdecl";
+    // filename = "d:/scratch/R5_H25_C1_aug_grid.grdecl";
+
+    std::string filename = qtFileName.toStdString();
+
+    auto objects = RifEclipseTextFileReader::readKeywordAndValues( filename );
+
+    EXPECT_EQ( size_t( 7 ), objects.size() );
+    for ( const auto& obj : objects )
+    {
+        std::cout << obj.keyword << "\n";
+    }
+}
+
+//--------------------------------------------------------------------------------------------------
+///
+//--------------------------------------------------------------------------------------------------
+TEST( RifEclipseTextFileReader, ReadLine )
+{
+    {
+        // Empty string
+
+        std::string fileContent;
+        size_t      offset    = 0;
+        size_t      bytesRead = 0;
+        auto        line      = RifEclipseTextFileReader::readLine( fileContent, offset, bytesRead );
+
+        EXPECT_EQ( size_t( 0 ), bytesRead );
+        EXPECT_EQ( size_t( 0 ), line.size() );
+    }
+
+    {
+        // Offset too large
+        std::string fileContent = "f";
+        size_t      offset      = 10;
+        size_t      bytesRead   = 0;
+        auto        line        = RifEclipseTextFileReader::readLine( fileContent, offset, bytesRead );
+
+        EXPECT_EQ( size_t( 0 ), bytesRead );
+        EXPECT_EQ( size_t( 0 ), line.size() );
+    }
+
+    {
+        // One line, no line break
+        std::string fileContent = "file content";
+        size_t      offset      = 0;
+        size_t      bytesRead   = 0;
+        auto        line        = RifEclipseTextFileReader::readLine( fileContent, offset, bytesRead );
+
+        EXPECT_EQ( size_t( 12 ), bytesRead );
+        EXPECT_EQ( size_t( 12 ), line.size() );
+    }
+
+    {
+        // two lines with line break
+        std::string fileContent = "file content\n next Line";
+        size_t      offset      = 0;
+        size_t      bytesRead   = 0;
+        auto        line        = RifEclipseTextFileReader::readLine( fileContent, offset, bytesRead );
+
+        // bytesRead includes line break
+        EXPECT_EQ( size_t( 13 ), bytesRead );
+        EXPECT_EQ( size_t( 12 ), line.size() );
+
+        auto secondLine = RifEclipseTextFileReader::readLine( fileContent, offset + bytesRead, bytesRead );
+        EXPECT_EQ( size_t( 10 ), bytesRead );
+        EXPECT_EQ( size_t( 9 ), secondLine.size() );
+    }
+}