OilfieldToolsNet/ResInsight
4
0
Fork 0
mirror of https://github.com/OPM/ResInsight.git synced 2025-02-25 18:55:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

Summary performance improvements

Browse Source 
* Check flag before rebuilding summary address nodes
* Performance: Use vector instead of set and map locally in thread
* Performance: Skip looking for restart summary filenames for opm-common
* Move adding of case realization parameters to OpenMP loop
* Add unit test for file path operations
* Performance: Avoid using cdUp() function when splitting path into strings
This commit is contained in:
Magne Sjaastad 2024-03-18 11:46:43 +01:00 committed by GitHub
parent 63defd0f1a
commit a880a42752
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 131 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
ApplicationLibCode/Application/Tools/RiaFilePathTools.cpp
View File

@ -204,9 +204,12 @@ QStringList RiaFilePathTools::splitPathIntoComponents( const QString& inputPath,
{
auto path = QDir::cleanPath( inputPath );
QStringList components;
auto indexOfLastSeparator = path.lastIndexOf( separator() );
auto indexOfDrive = path.indexOf( ':' );
QDir dir( path );
QString pathWithoutDrive = path.mid( indexOfDrive + 1, indexOfLastSeparator - ( indexOfDrive + 1 ) );
QStringList components = RiaTextStringTools::splitSkipEmptyParts( pathWithoutDrive, separator() );
QFileInfo fileInfo( path );
@ -214,18 +217,14 @@ QStringList RiaFilePathTools::splitPathIntoComponents( const QString& inputPath,
{
QString extension = fileInfo.completeSuffix();
path = path.replace( QString( ".%1" ).arg( extension ), "" );
components.push_front( extension );
components.push_front( fileInfo.baseName() );
components.push_back( extension );
components.push_back( fileInfo.baseName() );
}
else
{
components.push_back( fileInfo.fileName() );
}
while ( dir.cdUp() )
{
components.push_front( dir.dirName() );
}
return components;
}
@ -330,16 +329,10 @@ std::map<QString, QStringList> RiaFilePathTools::keyPathComponentsForEachFilePat
{
std::map<QString, QStringList> allComponents;
std::multiset<QString> allPathComponents;
for ( auto fileName : filePaths )
{
QStringList pathComponentsForFile = splitPathIntoComponents( fileName, true );
allComponents[fileName] = pathComponentsForFile;
for ( auto pathComponent : pathComponentsForFile )
{
allPathComponents.insert( pathComponent );
}
}
auto topNode = std::unique_ptr<PathNode>( new PathNode( "", nullptr ) );

10
ApplicationLibCode/FileInterface/RifOpmCommonSummary.cpp
View File

@ -398,9 +398,9 @@ std::pair<std::set<RifEclipseSummaryAddress>, std::map<RifEclipseSummaryAddress,
#pragma omp parallel
{
std::set<RifEclipseSummaryAddress> threadAddresses;
std::map<RifEclipseSummaryAddress, std::string> threadAddressToKeywordMap;
std::vector<std::string> threadInvalidKeywords;
std::vector<RifEclipseSummaryAddress> threadAddresses;
std::vector<std::pair<RifEclipseSummaryAddress, std::string>> threadAddressToKeywordMap;
std::vector<std::string> threadInvalidKeywords;
#pragma omp for
for ( int index = 0; index < (int)keywords.size(); index++ )
@ -418,8 +418,8 @@ std::pair<std::set<RifEclipseSummaryAddress>, std::map<RifEclipseSummaryAddress,
if ( eclAdr.isValid() )
{
threadAddresses.insert( eclAdr );
threadAddressToKeywordMap[eclAdr] = keyword;
threadAddresses.emplace_back( eclAdr );
threadAddressToKeywordMap.emplace_back( std::make_pair( eclAdr, keyword ) );
}
}

10
ApplicationLibCode/ProjectDataModel/RimSummaryCalculationCollection.cpp
View File

@ -58,6 +58,8 @@ void RimSummaryCalculationCollection::updateDataDependingOnCalculations()
{
if ( !summaryCase ) continue;
if ( !summaryCase->showRealizationDataSources() ) continue;
if ( auto reader = summaryCase->summaryReader() )
{
reader->buildMetaData();
@ -93,11 +95,3 @@ void RimSummaryCalculationCollection::rebuildCaseMetaData()
ensureValidCalculationIds();
updateDataDependingOnCalculations();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RimSummaryCalculationCollection::initAfterRead()
{
rebuildCaseMetaData();
}

1
ApplicationLibCode/ProjectDataModel/RimSummaryCalculationCollection.h
View File

@ -43,5 +43,4 @@ public:
private:
void updateDataDependingOnCalculations();
void initAfterRead() override;
};

23
ApplicationLibCode/ProjectDataModel/Summary/RimFileSummaryCase.cpp
View File

@ -20,14 +20,12 @@
#include "RiaApplication.h"
#include "RiaLogging.h"
#include "RicfCommandObject.h"
#include "RiaPreferencesSummary.h"
#include "RifEclipseSummaryTools.h"
#include "RifMultipleSummaryReaders.h"
#include "RifOpmCommonSummary.h"
#include "RifProjectSummaryDataWriter.h"
#include "RifReaderEclipseRft.h"
#include "RifReaderEclipseSummary.h"
#include "RifReaderOpmRft.h"
#include "RifSummaryReaderMultipleFiles.h"
@ -36,7 +34,6 @@
#include "RimProject.h"
#include "RimRftCase.h"
#include "RimSummaryCalculationCollection.h"
#include "RimTools.h"
#include "cafPdmFieldScriptingCapability.h"
#include "cafPdmObjectScriptingCapability.h"
@ -102,8 +99,18 @@ QString RimFileSummaryCase::caseName() const
//--------------------------------------------------------------------------------------------------
void RimFileSummaryCase::createSummaryReaderInterfaceThreadSafe( RiaThreadSafeLogger* threadSafeLogger )
{
m_fileSummaryReader =
RimFileSummaryCase::findRelatedFilesAndCreateReader( summaryHeaderFilename(), m_includeRestartFiles, threadSafeLogger );
bool lookForRestartFiles = false;
if ( RiaPreferencesSummary::current()->summaryDataReader() == RiaPreferencesSummary::SummaryReaderMode::LIBECL )
{
// It is only the libecl reader that requires manual search for referenced restart files
// opm-common reader handles restart files internally based on m_includeRestartFiles in RifOpmCommonEclipseSummary::openFileReader
//
// The performance of the function looking for restart files is bad, and will affect the performance significantly
lookForRestartFiles = m_includeRestartFiles;
}
m_fileSummaryReader = RimFileSummaryCase::findRelatedFilesAndCreateReader( summaryHeaderFilename(), lookForRestartFiles, threadSafeLogger );
m_multiSummaryReader = new RifMultipleSummaryReaders;
m_multiSummaryReader->addReader( m_fileSummaryReader.p() );
@ -159,10 +166,10 @@ void RimFileSummaryCase::createRftReaderInterface()
///
//--------------------------------------------------------------------------------------------------
RifSummaryReaderInterface* RimFileSummaryCase::findRelatedFilesAndCreateReader( const QString& headerFileName,
bool includeRestartFiles,
bool lookForRestartFiles,
RiaThreadSafeLogger* threadSafeLogger )
{
if ( includeRestartFiles )
if ( lookForRestartFiles )
{
std::vector<QString> warnings;
std::vector<RifRestartFileInfo> restartFileInfos = RifEclipseSummaryTools::getRestartFiles( headerFileName, warnings );

2
ApplicationLibCode/ProjectDataModel/Summary/RimFileSummaryCase.h
View File

@ -61,7 +61,7 @@ public:
void onProjectBeingSaved();
static RifSummaryReaderInterface*
findRelatedFilesAndCreateReader( const QString& headerFileName, bool includeRestartFiles, RiaThreadSafeLogger* threadSafeLogger );
findRelatedFilesAndCreateReader( const QString& headerFileName, bool lookForRestartFiles, RiaThreadSafeLogger* threadSafeLogger );
protected:
void defineEditorAttribute( const caf::PdmFieldHandle* field, QString uiConfigName, caf::PdmUiEditorAttribute* attribute ) override;

3
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryCase.cpp
View File

@ -406,6 +406,9 @@ void RimSummaryCase::refreshMetaData()
//--------------------------------------------------------------------------------------------------
void RimSummaryCase::onCalculationUpdated()
{
// NB! Performance critical method
if ( !m_showSubNodesInTree ) return;
// Delete all calculated address objects
m_dataVectorFolders->deleteCalculatedObjects();

4
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryCaseCollection.cpp
View File

@ -248,7 +248,9 @@ void RimSummaryCaseCollection::ensureNameIsUpdated()
RiaEnsembleNameTools::EnsembleGroupingMode groupingMode = RiaEnsembleNameTools::EnsembleGroupingMode::FMU_FOLDER_STRUCTURE;
QString ensembleName = RiaEnsembleNameTools::findSuitableEnsembleName( fileNames, groupingMode );
m_name = ensembleName;
if ( m_name == ensembleName ) return;
m_name = ensembleName;
caseNameChanged.send();
}
}

10
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryCaseMainCollection.cpp
View File

@ -460,6 +460,7 @@ void RimSummaryCaseMainCollection::loadFileSummaryCaseData( std::vector<RimFileS
if ( fileSummaryCase )
{
fileSummaryCase->createSummaryReaderInterfaceThreadSafe( &threadSafeLogger );
addCaseRealizationParametersIfFound( *fileSummaryCase, fileSummaryCase->summaryHeaderFilename() );
}
progInfo.setProgress( cIdx );
@ -482,7 +483,6 @@ void RimSummaryCaseMainCollection::loadFileSummaryCaseData( std::vector<RimFileS
if ( fileSummaryCase )
{
fileSummaryCase->createRftReaderInterface();
addCaseRealizationParametersIfFound( *fileSummaryCase, fileSummaryCase->summaryHeaderFilename() );
}
}
}
@ -648,9 +648,13 @@ void RimSummaryCaseMainCollection::updateAutoShortName()
//
// https://github.com/OPM/ResInsight/issues/7438
for ( auto s : allSummaryCases() )
auto sumCases = allSummaryCases();
#pragma omp parallel for
for ( int cIdx = 0; cIdx < static_cast<int>( sumCases.size() ); ++cIdx )
{
s->updateAutoShortName();
auto sumCase = sumCases[cIdx];
sumCase->updateAutoShortName();
}
}

88
ApplicationLibCode/UnitTests/RiaFilePathTools-Test.cpp
View File

@ -1,5 +1,6 @@
#include "gtest/gtest.h"
#include "RiaEnsembleNameTools.h"
#include "RiaFilePathTools.h"
#include <iostream>
@ -112,3 +113,90 @@ TEST( RiaFilePathTools, removeDuplicatePathSeparators )
EXPECT_STRCASEEQ( expectedPath.toLatin1(), resultRootPath.toLatin1() );
}
}
//--------------------------------------------------------------------------------------------------
TEST( RiaFilePathTools, splitIntoComponets )
{
{
QString testPath( "e:/models/from_equinor_sftp/drogon3d_ahm/realization-0/iter-3/eclipse/model/DROGON-0.SMSPEC" );
auto words = RiaFilePathTools::splitPathIntoComponents( testPath );
EXPECT_EQ( 8, words.size() );
EXPECT_EQ( QString( "models" ), words[0] );
EXPECT_EQ( QString( "from_equinor_sftp" ), words[1] );
EXPECT_EQ( QString( "drogon3d_ahm" ), words[2] );
EXPECT_EQ( QString( "realization-0" ), words[3] );
EXPECT_EQ( QString( "iter-3" ), words[4] );
EXPECT_EQ( QString( "eclipse" ), words[5] );
EXPECT_EQ( QString( "model" ), words[6] );
EXPECT_EQ( QString( "DROGON-0.SMSPEC" ), words[7] );
}
{
QString testPath( "/home/builder/models/realization-0/iter-3/eclipse/model/DROGON-0.SMSPEC" );
auto words = RiaFilePathTools::splitPathIntoComponents( testPath );
EXPECT_EQ( 8, words.size() );
EXPECT_EQ( QString( "home" ), words[0] );
EXPECT_EQ( QString( "builder" ), words[1] );
EXPECT_EQ( QString( "models" ), words[2] );
EXPECT_EQ( QString( "realization-0" ), words[3] );
EXPECT_EQ( QString( "iter-3" ), words[4] );
EXPECT_EQ( QString( "eclipse" ), words[5] );
EXPECT_EQ( QString( "model" ), words[6] );
EXPECT_EQ( QString( "DROGON-0.SMSPEC" ), words[7] );
}
}
//--------------------------------------------------------------------------------------------------
TEST( RiaFilePathTools, EnsembleName )
{
{
QString testPath1( "e:/models/from_equinor_sftp/drogon3d_ahm/realization-0/iter-3/eclipse/model/DROGON-0.SMSPEC" );
QString testPath2( "e:/models/from_equinor_sftp/drogon3d_ahm/realization-1/iter-3/eclipse/model/DROGON-1.SMSPEC" );
QStringList allPaths = { testPath1, testPath2 };
auto ensembleName =
RiaEnsembleNameTools::findSuitableEnsembleName( allPaths, RiaEnsembleNameTools::EnsembleGroupingMode::FMU_FOLDER_STRUCTURE );
EXPECT_EQ( QString( "iter-3" ), ensembleName );
}
}
//--------------------------------------------------------------------------------------------------
TEST( RiaFilePathTools, RealizationName )
{
{
QString testPath0( "e:/models/from_equinor_sftp/drogon3d_ahm/realization-0/iter-3/eclipse/model/DROGON-0.SMSPEC" );
QString testPath1( "e:/models/from_equinor_sftp/drogon3d_ahm/realization-1/iter-3/eclipse/model/DROGON-1.SMSPEC" );
QStringList allPaths = { testPath0, testPath1 };
QString fileName = "DROGON-0.SMSPEC";
auto name = RiaEnsembleNameTools::uniqueShortName( testPath1, allPaths, fileName );
EXPECT_EQ( QString( "real-1" ), name );
}
}
//--------------------------------------------------------------------------------------------------
TEST( RiaFilePathTools, keyPathComponentsForEachFilePath )
{
{
QString testPath0( "e:/models/from_equinor_sftp/drogon3d_ahm/realization-0/iter-3/eclipse/model/DROGON-0.SMSPEC" );
QString testPath1( "e:/models/from_equinor_sftp/drogon3d_ahm/realization-1/iter-3/eclipse/model/DROGON-1.SMSPEC" );
QStringList allPaths = { testPath0, testPath1 };
auto keyComponents = RiaFilePathTools::keyPathComponentsForEachFilePath( allPaths );
auto test0 = keyComponents[testPath0];
EXPECT_EQ( QString( "realization-0" ), test0.front() );
auto test1 = keyComponents[testPath1];
EXPECT_EQ( QString( "realization-1" ), test1.front() );
}
}