OilfieldToolsNet/ResInsight
4
0
Fork 0
mirror of https://github.com/OPM/ResInsight.git synced 2025-02-25 18:55:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

#7563 HDF5 Export : Add support for defining how many threads to use

Browse Source
This commit is contained in:
Magne Sjaastad
2021-04-13 19:19:55 +02:00
parent 6bc6ffda15
commit 27e3bc1b56
8 changed files with 181 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
ApplicationLibCode/Application/RiaPreferencesSummary.cpp
View File

@@ -1,25 +1,25 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2021 Equinor ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#include "RiaPreferencesSummary.h"
#include "cafPdmUiCheckBoxEditor.h"
// #include "RiaColorTables.h"
// #include "RiaValidRegExpValidator.h"
// #include "RifReaderSettings.h"
// #include "RiuGuiTheme.h"
//
// #include "cafPdmFieldCvfColor.h"
// #include "cafPdmSettings.h"
// #include "cafPdmUiCheckBoxEditor.h"
// #include "cafPdmUiComboBoxEditor.h"
// #include "cafPdmUiFieldHandle.h"
// #include "cafPdmUiFilePathEditor.h"
// #include "cafPdmUiLineEditor.h"
//
// #include <QDate>
// #include <QDir>
// #include <QLocale>
// #include <QRegExp>
// #include <QStandardPaths>
#include <algorithm>
namespace caf
{
@@ -109,6 +109,14 @@ RiaPreferencesSummary::RiaPreferencesSummary()
"" );
m_createH5SummaryDataFile.uiCapability()->setUiLabelPosition( caf::PdmUiItemInfo::HIDDEN );
CAF_PDM_InitField( &m_createH5SummaryFileThreadCount,
"createH5SummaryFileThreadCount",
4,
"H5 Summary Data Creation Thread Count [BETA]",
"",
"",
"" );
CAF_PDM_InitFieldNoDefault( &m_summaryReader, "summaryReaderType", "Summary Data File Reader", "", "", "" );
}
@@ -144,6 +152,16 @@ bool RiaPreferencesSummary::createH5SummaryDataFiles() const
return m_createH5SummaryDataFile();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
int RiaPreferencesSummary::createH5SummaryDataThreadCount() const
{
const int minimumThreadCount = 1;
return std::max( minimumThreadCount, m_createH5SummaryFileThreadCount() );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
@@ -177,6 +195,7 @@ void RiaPreferencesSummary::defineUiOrdering( QString uiConfigName, caf::PdmUiOr
else if ( m_summaryReader == SummaryReaderMode::HDF5_OPM_COMMON )
{
uiOrdering.add( &m_createH5SummaryDataFile );
uiOrdering.add( &m_createH5SummaryFileThreadCount );
}
uiOrdering.skipRemainingFields();

46
ApplicationLibCode/Application/RiaPreferencesSummary.h
View File

@@ -1,25 +1,26 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2021 Equinor ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#pragma once
// #include "RiaDefines.h"
// #include "RiaFontCache.h"
// #include "RiaGuiApplication.h"
// #include "RiaQDateTimeTools.h"
//
// #include "cafAppEnum.h"
// #include "cafPdmChildField.h"
// #include "cafPdmField.h"
// #include "cafPdmObject.h"
//
// // Include to make Pdm work for cvf::Color
// #include "cafPdmFieldCvfColor.h"
//
// #include <QPageLayout>
// #include <QPageSize>
// #include <QStringList>
//
// #include <map>
//
// class RifReaderSettings;
#include "cafAppEnum.h"
#include "cafPdmField.h"
#include "cafPdmObject.h"
//--------------------------------------------------------------------------------------------------
///
@@ -43,7 +44,9 @@ public:
SummaryReaderMode summaryDataReader() const;
bool useOptimizedSummaryDataFiles() const;
bool createOptimizedSummaryDataFiles() const;
bool createH5SummaryDataFiles() const;
bool createH5SummaryDataFiles() const;
int createH5SummaryDataThreadCount() const;
protected:
void defineEditorAttribute( const caf::PdmFieldHandle* field,
@@ -58,6 +61,7 @@ private:
caf::PdmField<bool> m_useOptimizedSummaryDataFile;
caf::PdmField<bool> m_createH5SummaryDataFile;
caf::PdmField<int> m_createH5SummaryFileThreadCount;
caf::PdmField<SummaryReaderModeType> m_summaryReader;
};

53
ApplicationLibCode/FileInterface/RifHdf5SummaryExporter.cpp
View File

@@ -34,7 +34,47 @@
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
bool RifHdf5SummaryExporter::ensureHdf5FileIsCreated( const std::string& smspecFileName, const std::string& h5FileName )
bool RifHdf5SummaryExporter::ensureHdf5FileIsCreatedMultithreaded( const std::vector<std::string>& smspecFileNames,
const std::vector<std::string>& h5FileNames,
int threadCount )
{
if ( smspecFileNames.empty() ) return true;
if ( smspecFileNames.size() != h5FileNames.size() ) return false;
{
QString txt = QString( "Testing if H5 files are present for [ %1 ] summary files ..." )
.arg( static_cast<int>( smspecFileNames.size() ) );
RiaLogging::info( txt );
}
size_t hdfFilesCreatedCount = 0;
#pragma omp parallel for schedule( dynamic ) num_threads( threadCount )
for ( int cIdx = 0; cIdx < static_cast<int>( smspecFileNames.size() ); ++cIdx )
{
auto smspecFileName = smspecFileNames[cIdx];
auto h5FileName = h5FileNames[cIdx];
RifHdf5SummaryExporter::ensureHdf5FileIsCreated( smspecFileName, h5FileName, hdfFilesCreatedCount );
}
{
QString txt = QString( "Created [ %1 ] h5 files from a total of [ %2 ] summary files" )
.arg( static_cast<int>( hdfFilesCreatedCount ) )
.arg( static_cast<int>( smspecFileNames.size() ) );
RiaLogging::info( txt );
}
return true;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
bool RifHdf5SummaryExporter::ensureHdf5FileIsCreated( const std::string& smspecFileName,
const std::string& h5FileName,
size_t& hdfFilesCreatedCount )
{
if ( !QFile::exists( QString::fromStdString( smspecFileName ) ) ) return false;
@@ -49,10 +89,15 @@ bool RifHdf5SummaryExporter::ensureHdf5FileIsCreated( const std::string& smspecF
// performance penalty
sourceSummaryData.LoadData();
RifHdf5Exporter exporter( h5FileName );
#pragma omp critical( critical_section_HDF5_export )
{
RifHdf5Exporter exporter( h5FileName );
writeGeneralSection( exporter, sourceSummaryData );
writeSummaryVectors( exporter, sourceSummaryData );
writeGeneralSection( exporter, sourceSummaryData );
writeSummaryVectors( exporter, sourceSummaryData );
hdfFilesCreatedCount++;
}
}
catch ( std::exception& e )
{

8
ApplicationLibCode/FileInterface/RifHdf5SummaryExporter.h
View File

@@ -40,7 +40,13 @@ namespace EclIO
class RifHdf5SummaryExporter
{
public:
static bool ensureHdf5FileIsCreated( const std::string& smspecFileName, const std::string& h5FileName );
static bool ensureHdf5FileIsCreatedMultithreaded( const std::vector<std::string>& smspecFileNames,
const std::vector<std::string>& h5FileNames,
int threadCount );
static bool ensureHdf5FileIsCreated( const std::string& smspecFileName,
const std::string& h5FileName,
size_t& hdfFilesCreatedCount );
private:
static bool writeGeneralSection( RifHdf5Exporter& exporter, Opm::EclIO::ESmry& sourceSummaryData );

4
ApplicationLibCode/FileInterface/RifReaderEclipseSummary.cpp
View File

@@ -171,8 +171,10 @@ bool RifReaderEclipseSummary::open( const QString& headerFileName,
QFileInfo fi( headerFileName );
QString h5FilenameCandidate = fi.absolutePath() + "/" + fi.baseName() + ".h5";
size_t dummy = 0;
RifHdf5SummaryExporter::ensureHdf5FileIsCreated( headerFileName.toStdString(),
h5FilenameCandidate.toStdString() );
h5FilenameCandidate.toStdString(),
dummy );
}
auto hdfReader = std::make_unique<RifOpmHdf5Summary>();

58
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryCaseMainCollection.cpp
View File

@@ -27,6 +27,10 @@
#include "RifOpmCommonSummary.h"
#include "RifSummaryCaseRestartSelector.h"
#ifdef USE_HDF5
#include "RifHdf5SummaryExporter.h"
#endif
#include "RimCaseDisplayNameTools.h"
#include "RimDerivedEnsembleCaseCollection.h"
#include "RimEclipseResultCase.h"
@@ -42,6 +46,10 @@
#include "cafProgressInfo.h"
#ifdef USE_OPENMP
#include <omp.h>
#endif
#include <QDir>
CAF_PDM_SOURCE_INIT( RimSummaryCaseMainCollection, "SummaryCaseCollection" );
@@ -408,7 +416,21 @@ void RimSummaryCaseMainCollection::loadSummaryCaseData( std::vector<RimSummaryCa
if ( !fileSummaryCases.empty() )
{
loadFileSummaryCaseData( fileSummaryCases );
auto prefSummary = RiaPreferences::current()->summaryPreferences();
int threadCount = 1;
#ifdef USE_OPENMP
if ( prefSummary->summaryDataReader() != RiaPreferencesSummary::SummaryReaderMode::HDF5_OPM_COMMON )
{
threadCount = prefSummary->createH5SummaryDataThreadCount();
}
else
{
threadCount = omp_get_max_threads();
}
#endif
loadFileSummaryCaseData( fileSummaryCases, threadCount );
}
caf::ProgressInfo progInfo( otherSummaryCases.size(), "Loading Summary Cases" );
@@ -432,20 +454,47 @@ void RimSummaryCaseMainCollection::loadSummaryCaseData( std::vector<RimSummaryCa
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RimSummaryCaseMainCollection::loadFileSummaryCaseData( std::vector<RimFileSummaryCase*> fileSummaryCases )
void RimSummaryCaseMainCollection::loadFileSummaryCaseData( std::vector<RimFileSummaryCase*> fileSummaryCases,
int threadCountForHdf5Export )
{
// Use openMP when reading file summary case meta data. Avoid using the virtual interface of base class
// RimSummaryCase, as it is difficult to make sure all variants of the leaf classes are thread safe.
// Only open the summary file reader in parallel loop to reduce risk of multi threading issues
RiaPreferencesSummary* prefSummary = RiaPreferences::current()->summaryPreferences();
#ifdef USE_HDF5
{
if ( prefSummary->summaryDataReader() == RiaPreferencesSummary::SummaryReaderMode::HDF5_OPM_COMMON &&
prefSummary->createH5SummaryDataFiles() )
{
std::vector<std::string> headerFileNames;
std::vector<std::string> h5FileNames;
for ( const auto fileSummaryCase : fileSummaryCases )
{
auto headerFileName = fileSummaryCase->summaryHeaderFilename();
QFileInfo fi( headerFileName );
QString h5FilenameCandidate = fi.absolutePath() + "/" + fi.baseName() + ".h5";
headerFileNames.push_back( headerFileName.toStdString() );
h5FileNames.push_back( h5FilenameCandidate.toStdString() );
}
RifHdf5SummaryExporter::ensureHdf5FileIsCreatedMultithreaded( headerFileNames,
h5FileNames,
threadCountForHdf5Export );
}
}
#endif
caf::ProgressInfo progInfo( fileSummaryCases.size(), "Loading Summary Cases" );
RifOpmCommonEclipseSummary::resetLodCount();
RiaThreadSafeLogger threadSafeLogger;
auto prefSummary = RiaPreferences::current()->summaryPreferences();
// The HDF5 reader requires a special configuration to be thread safe. Disable threading for HDF reader creation.
bool canUseMultipleTreads =
( prefSummary->summaryDataReader() != RiaPreferencesSummary::SummaryReaderMode::HDF5_OPM_COMMON );
@@ -474,6 +523,7 @@ void RimSummaryCaseMainCollection::loadFileSummaryCaseData( std::vector<RimFileS
.arg( numberOfLodFilesCreated ) );
}
// This loop is not thread safe, use serial loop
for ( int cIdx = 0; cIdx < static_cast<int>( fileSummaryCases.size() ); ++cIdx )
{
RimFileSummaryCase* fileSummaryCase = fileSummaryCases[cIdx];

2
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryCaseMainCollection.h
View File

@@ -74,7 +74,7 @@ public:
private:
static void loadSummaryCaseData( std::vector<RimSummaryCase*> summaryCases );
static void loadFileSummaryCaseData( std::vector<RimFileSummaryCase*> fileSummaryCases );
static void loadFileSummaryCaseData( std::vector<RimFileSummaryCase*> fileSummaryCases, int threadCountForHdf5Export );
static void reassignSummaryCurves( const RimGridSummaryCase* fromGridCase, RimFileSummaryCase* toFileCase );
static RimSummaryCaseCollection* defaultAllocator();

8
ApplicationLibCode/UnitTests/HDF5FileWriter-Test.cpp
View File

@@ -25,10 +25,12 @@ TEST( DISABLED_HDFTests, WriteToHdf5SummaryExporter )
{
QString file_path = H5_TEST_DATA_DIRECTORY_2 + "NORNE_ATW2013_RFTPLT_V2.SMSPEC";
RifHdf5SummaryExporter exporter;
std::string exportFileName = "e:/project/scratch_export/hdf_complete.h5";
std::string exportFileName = "e:/project/scratch_export/hdf_complete.h5";
exporter.ensureHdf5FileIsCreated( file_path.toStdString(), exportFileName );
int threadCount = 1;
RifHdf5SummaryExporter::ensureHdf5FileIsCreatedMultithreaded( { file_path.toStdString() },
{ exportFileName },
threadCount );
}
//--------------------------------------------------------------------------------------------------