From f708ac96f2b3357e22a43ec60ab0d211dfb5e379 Mon Sep 17 00:00:00 2001 From: Magne Sjaastad Date: Sat, 3 May 2025 09:02:07 +0200 Subject: [PATCH] Add path pattern helper functions and tests --- .../Tools/Ensemble/RiaEnsembleImportTools.cpp | 213 ++++++++++++++++++ .../Tools/Ensemble/RiaEnsembleImportTools.h | 3 + ApplicationLibCode/UnitTests/CMakeLists.txt | 1 + .../UnitTests/RiaEnsembleImportTools-Test.cpp | 72 ++++++ 4 files changed, 289 insertions(+) create mode 100644 ApplicationLibCode/UnitTests/RiaEnsembleImportTools-Test.cpp diff --git a/ApplicationLibCode/Application/Tools/Ensemble/RiaEnsembleImportTools.cpp b/ApplicationLibCode/Application/Tools/Ensemble/RiaEnsembleImportTools.cpp index f2cc52c340..81cb04e355 100644 --- a/ApplicationLibCode/Application/Tools/Ensemble/RiaEnsembleImportTools.cpp +++ b/ApplicationLibCode/Application/Tools/Ensemble/RiaEnsembleImportTools.cpp @@ -20,12 +20,16 @@ #include "RiaGuiApplication.h" #include "RiaLogging.h" +#include "RiaStdStringTools.h" #include "Summary/RiaSummaryTools.h" #include "RifSummaryCaseRestartSelector.h" #include "RimSummaryCaseMainCollection.h" +#include +#include + namespace RiaEnsembleImportTools { @@ -78,4 +82,213 @@ std::pair> createSummaryCasesFromFiles( const return std::make_pair( true, newCases ); } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::pair findPathPattern( const QStringList& filePaths, const QString& placeholderString ) +{ + if ( filePaths.isEmpty() ) return {}; + + std::vector> tableOfNumbers; + + QRegularExpression numberRegex( "\\d+" ); + + for ( const auto& f : filePaths ) + { + std::vector valuesInString; + auto matchIterator = numberRegex.globalMatch( f ); + while ( matchIterator.hasNext() ) + { + auto match = matchIterator.next(); + QString number = match.captured( 0 ); + + valuesInString.push_back( number.toInt() ); + } + tableOfNumbers.push_back( valuesInString ); + } + + if ( tableOfNumbers.empty() ) return {}; + + const auto valuesFirstRow = tableOfNumbers[0]; + if ( valuesFirstRow.empty() ) return {}; + + std::vector valueCountEachIndex; + valueCountEachIndex.resize( valuesFirstRow.size(), 1 ); + + for ( size_t rowIndex = 1; rowIndex < tableOfNumbers.size(); ++rowIndex ) + { + const auto& values = tableOfNumbers[rowIndex]; + for ( size_t j = 0; j < std::min( valuesFirstRow.size(), values.size() ); ++j ) + { + if ( values[j] != valuesFirstRow[j] ) + { + valueCountEachIndex[j]++; + } + } + } + + QString firstPath = filePaths[0]; + QList> numberPositions; // start pos, length + + auto i = numberRegex.globalMatch( firstPath ); + while ( i.hasNext() ) + { + auto match = i.next(); + numberPositions.append( qMakePair( match.capturedStart(), match.capturedLength() ) ); + } + + if ( numberPositions.isEmpty() ) return {}; + + // For each number position, check if there are unique values for all rows + QList> varyingNumberPositions; + + for ( size_t i = 0; i < static_cast( numberPositions.size() ); i++ ) + { + const auto& varyPos = numberPositions[i]; + if ( i < valueCountEachIndex.size() && valueCountEachIndex[i] == tableOfNumbers.size() ) + { + varyingNumberPositions.push_back( varyPos ); + } + } + + if ( varyingNumberPositions.empty() ) return {}; + + // Extract the varying numbers + std::vector numbers; + + // Find the first varying position + int varyingPosition = -1; + for ( size_t i = 0; i < valueCountEachIndex.size(); i++ ) + { + if ( valueCountEachIndex[i] == tableOfNumbers.size() ) + { + varyingPosition = static_cast( i ); + break; + } + } + + // Extract numbers from the varying position + if ( varyingPosition >= 0 ) + { + for ( const auto& values : tableOfNumbers ) + { + if ( varyingPosition < static_cast( values.size() ) ) + { + numbers.push_back( values[varyingPosition] ); + } + } + } + + if ( numbers.size() != static_cast( filePaths.size() ) ) return {}; + + auto pattern = filePaths.front(); + + // Loop over varying number positions in reverse order to avoid messing up the positions when replacing the varying part with a + // placeholder + for ( int i = varyingNumberPositions.size() - 1; i >= 0; --i ) + { + const auto& [start, length] = varyingNumberPositions[i]; + + pattern.replace( start, length, placeholderString ); + } + + auto rangeString = QString::fromStdString( RiaStdStringTools::formatRangeSelection( numbers ) ); + + return { pattern, rangeString }; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QStringList createPathsFromPattern( const QString& basePath, const QString& numberRange, const QString& placeholderString ) +{ + QStringList paths; + + auto numbers = RiaStdStringTools::valuesFromRangeSelection( numberRange.toStdString() ); + for ( const auto& number : numbers ) + { + QString path = basePath; + path.replace( placeholderString, QString::number( number ) ); + paths.push_back( path ); + } + + return paths; +} + +//-------------------------------------------------------------------------------------------------- +/// Find all files matching the path pattern in the file system +// +// Example: +// pathPattern = "/myfolder/drogon-varying-grid-geometry/realization-*/iter-0/eclipse/model/DROGON-*.EGRID" +// +// basePath will be "/myfolder/drogon-varying-grid-geometry/" +// +// The '*' in the pathPattern is replaced with a regex pattern to capture numbers +// regexPattern will be "/myfolder/drogon-varying-grid-geometry/realization-(\\d+)/iter-0/eclipse/model/DROGON-(\\d+).EGRID" +// +// The basePath will be traversed recursively to find all files matching the regex pattern +// +//-------------------------------------------------------------------------------------------------- +QStringList createPathsBySearchingFileSystem( const QString& pathPattern, const QString& placeholderString ) +{ + auto basePath = pathPattern; + + // Find based path up to "/realization" + auto realizationPos = basePath.indexOf( "/realization" ); + if ( realizationPos != -1 ) + { + basePath = basePath.left( realizationPos ); + } + + QStringList matchingFiles; + + // Replace placeholder string with a regex pattern to capture numbers + QString regexPattern = pathPattern; + regexPattern.replace( placeholderString, "(\\d+)" ); + + QRegularExpression regex( regexPattern ); + + // Use QDirIterator to traverse the directory recursively + QDirIterator it( basePath, QDir::Files, QDirIterator::Subdirectories ); + while ( it.hasNext() ) + { + QString filePath = it.next(); + + QRegularExpressionMatch match = regex.match( filePath ); + if ( match.hasMatch() ) + { + QString index1 = match.captured( 1 ); + QString index2 = match.captured( 2 ); + + if ( index1 == index2 ) + { + matchingFiles << filePath; + } + } + } + + // Sort files by realization number + std::sort( matchingFiles.begin(), + matchingFiles.end(), + []( const QString& a, const QString& b ) + { + QRegularExpression regex( "realization-(\\d+)" ); + + auto matchA = regex.match( a ); + auto matchB = regex.match( b ); + + if ( matchA.hasMatch() && matchB.hasMatch() ) + { + int numA = matchA.captured( 1 ).toInt(); + int numB = matchB.captured( 1 ).toInt(); + return numA < numB; + } + + // Fallback to alphabetical if regex doesn't match + return a < b; + } ); + + return matchingFiles; +} + } // namespace RiaEnsembleImportTools diff --git a/ApplicationLibCode/Application/Tools/Ensemble/RiaEnsembleImportTools.h b/ApplicationLibCode/Application/Tools/Ensemble/RiaEnsembleImportTools.h index 31c2081ce5..d0bfaeb652 100644 --- a/ApplicationLibCode/Application/Tools/Ensemble/RiaEnsembleImportTools.h +++ b/ApplicationLibCode/Application/Tools/Ensemble/RiaEnsembleImportTools.h @@ -34,4 +34,7 @@ struct CreateConfig }; std::pair> createSummaryCasesFromFiles( const QStringList& fileNames, CreateConfig createConfig ); +std::pair findPathPattern( const QStringList& filePaths, const QString& placeholderString ); +QStringList createPathsFromPattern( const QString& basePath, const QString& numberRange, const QString& placeholderString ); +QStringList createPathsBySearchingFileSystem( const QString& pathPattern, const QString& placeholderString ); } // namespace RiaEnsembleImportTools diff --git a/ApplicationLibCode/UnitTests/CMakeLists.txt b/ApplicationLibCode/UnitTests/CMakeLists.txt index f94883e939..c7ed9bca04 100644 --- a/ApplicationLibCode/UnitTests/CMakeLists.txt +++ b/ApplicationLibCode/UnitTests/CMakeLists.txt @@ -109,6 +109,7 @@ set(SOURCE_UNITTEST_FILES ${CMAKE_CURRENT_LIST_DIR}/RigPolygonTools-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RifVtkSurfaceImporter-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaKeyValueStore-Test.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaEnsembleImportTools-Test.cpp ) if(RESINSIGHT_ENABLE_GRPC) diff --git a/ApplicationLibCode/UnitTests/RiaEnsembleImportTools-Test.cpp b/ApplicationLibCode/UnitTests/RiaEnsembleImportTools-Test.cpp new file mode 100644 index 0000000000..fe810aa7c0 --- /dev/null +++ b/ApplicationLibCode/UnitTests/RiaEnsembleImportTools-Test.cpp @@ -0,0 +1,72 @@ +#include "gtest/gtest.h" + +#include "Tools/Ensemble/RiaEnsembleImportTools.h" + +namespace internal +{ +QString placeholderText() +{ + return "$(INDEX)"; +} +} // namespace internal + +TEST( RimPathPatternFileSetTest, OneVaryingNumberRealizations ) +{ + QStringList filePaths = { "drogon-varying-grid-geometry/realization-0/iter-0/eclipse/model/DROGON-0.EGRID", + "drogon-varying-grid-geometry/realization-1/iter-0/eclipse/model/DROGON-1.EGRID", + "drogon-varying-grid-geometry/realization-2/iter-0/eclipse/model/DROGON-2.EGRID", + "drogon-varying-grid-geometry/realization-3/iter-0/eclipse/model/DROGON-3.EGRID", + "drogon-varying-grid-geometry/realization-13/iter-0/eclipse/model/DROGON-13.EGRID" }; + + const auto& [basePath, numberRange] = RiaEnsembleImportTools::findPathPattern( filePaths, internal::placeholderText() ); + EXPECT_STREQ( basePath.toStdString().data(), + "drogon-varying-grid-geometry/realization-$(INDEX)/iter-0/eclipse/model/DROGON-$(INDEX).EGRID" ); + EXPECT_STREQ( numberRange.toStdString().data(), "0-3, 13" ); + + const auto paths = RiaEnsembleImportTools::createPathsFromPattern( basePath, numberRange, internal::placeholderText() ); + EXPECT_EQ( paths.size(), 5 ); + if ( paths.size() != filePaths.size() ) return; + for ( auto i = 0; i < paths.size(); i++ ) + { + EXPECT_STREQ( paths[i].toStdString().data(), filePaths[i].toStdString().data() ); + } +} + +TEST( RimPathPatternFileSetTest, OneVaryingNumberMultipleLocations ) +{ + QStringList filePaths = { "file_1/path-02/real-1.txt", "file_2/path-02/real-2.txt", "file_3/path-02/real-3.txt", "file_13/path-02/real-13.txt" }; + const auto& [basePath, numberRange] = RiaEnsembleImportTools::findPathPattern( filePaths, internal::placeholderText() ); + EXPECT_STREQ( basePath.toStdString().data(), "file_$(INDEX)/path-02/real-$(INDEX).txt" ); + EXPECT_STREQ( numberRange.toStdString().data(), "1-3, 13" ); + + const auto paths = RiaEnsembleImportTools::createPathsFromPattern( basePath, numberRange, internal::placeholderText() ); + EXPECT_EQ( paths.size(), 4 ); + EXPECT_STREQ( paths[0].toStdString().data(), "file_1/path-02/real-1.txt" ); + EXPECT_STREQ( paths[1].toStdString().data(), "file_2/path-02/real-2.txt" ); + EXPECT_STREQ( paths[2].toStdString().data(), "file_3/path-02/real-3.txt" ); + EXPECT_STREQ( paths[3].toStdString().data(), "file_13/path-02/real-13.txt" ); +} + +TEST( RimPathPatternFileSetTest, OneVaryingNumber ) +{ + QStringList filePaths = { "file_001.txt", "file_002.txt", "file_003.txt" }; + auto result = RiaEnsembleImportTools::findPathPattern( filePaths, internal::placeholderText() ); + EXPECT_STREQ( result.first.toStdString().data(), "file_$(INDEX).txt" ); + EXPECT_STREQ( result.second.toStdString().data(), "1-3" ); +} + +TEST( RimPathPatternFileSetTest, EmptyInput ) +{ + QStringList filePaths; + auto result = RiaEnsembleImportTools::findPathPattern( filePaths, internal::placeholderText() ); + EXPECT_TRUE( result.first.isEmpty() ); + EXPECT_TRUE( result.second.isEmpty() ); +} + +TEST( RimPathPatternFileSetTest, NoVaryingNumbers ) +{ + QStringList filePaths = { "file_123.txt", "file_123.txt" }; + auto result = RiaEnsembleImportTools::findPathPattern( filePaths, internal::placeholderText() ); + EXPECT_TRUE( result.first.isEmpty() ); + EXPECT_TRUE( result.second.isEmpty() ); +}