OilfieldToolsNet/ResInsight
4
0
Fork 0
mirror of https://github.com/OPM/ResInsight.git synced 2026-08-27 05:37:21 -05:00
Code Issues Packages Projects Releases Wiki Activity

Improved Sumo summary data fetching (#14551)

Browse Source 
Sumo data is loaded so the application stays responsive while it waits.

Run Sumo transfers on a dedicated thread
Split RiaSumoConnector into transport and data delegates
Load Sumo summary vectors concurrently and without waiting
Show that Sumo data is being loaded
This commit is contained in:
Jørgen Herje
2026-08-20 11:48:25 +02:00
committed by GitHub
parent 9b268e6d98
commit 0d9d4fed19
30 changed files with 2124 additions and 920 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake
+2
View File
@@ -3,6 +3,8 @@ set(SOURCE_GROUP_SOURCE_FILES
${CMAKE_CURRENT_LIST_DIR}/RiaCloudConnector.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaSumoConnector.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaSumoDefines.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaSumoExplore.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaSumoSummary.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaConnectorTools.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaOsduConnector.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaOAuthHttpServerReplyHandler.cpp
ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp
+466 -722
View File
File diff suppressed because it is too large Load Diff
ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h
+72 -97
View File
@@ -20,50 +20,23 @@
#include "RiaCloudConnector.h"
#include "RiaSumoDefines.h"
#include "RiaSumoExplore.h"
#include "RiaSumoSummary.h"
#include <QByteArray>
#include <QMutex>
#include <QNetworkAccessManager>
#include <QtNetworkAuth/QOAuth2AuthorizationCodeFlow>
#include <functional>
#include <list>
#include <map>
class QEventLoop;
class QThread;
using SumoObjectId = QString;
struct SumoAsset
{
SumoAssetId assetId;
QString kind;
QString name;
};
struct SumoCase
{
SumoCaseId caseId;
QString kind;
QString name;
};
struct SumoRedirect
{
SumoObjectId objectId;
QString blobName;
QString url;
QString redirectBaseUri;
QString redirectAuth;
QByteArray contents;
};
struct SumoEnsemble
{
SumoCaseId caseId;
QString name;
};
//==================================================================================================
///
//==================================================================================================
@@ -84,85 +57,87 @@ public:
QString server() const override;
void requestAssets();
void requestAssetsBlocking();
// Download blobs by id and return their contents. Getting a blob takes two round trips, one for the
// pre-signed URI and one for the data, and a batch does each of those as one concurrent group.
QByteArray downloadBlobBlocking( const QString& blobId );
std::map<QString, QByteArray> downloadBlobsBlocking( const std::vector<QString>& blobIds );
void requestCasesForField( const QString& fieldName );
void requestCasesForFieldBlocking( const QString& fieldName );
// What Sumo holds: assets, cases, ensembles and realizations.
RiaSumoExplore& explore();
void requestEnsembleByCasesId( const SumoCaseId& caseId );
void requestEnsembleByCasesIdBlocking( const SumoCaseId& caseId );
// The summary data of a case.
RiaSumoSummary& summary();
void requestVectorNamesForEnsemble( const SumoCaseId& caseId, const QString& ensembleName );
void requestVectorNamesForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName );
// Transport used by the data specific delegates. Every request goes through the transfer thread, so
// the calling thread waits without dispatching events.
QByteArray getBlocking( const QString& url, const QString& progressText = {} );
void requestRealizationIdsForEnsemble( const SumoCaseId& caseId, const QString& ensembleName );
void requestRealizationIdsForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName );
// The REST API returns a blob id as a plain string, quoted by FastAPI.
static QString blobIdFromBody( const QByteArray& body );
void requestParametersBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName );
void requestParametersBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName );
QByteArray requestParametersParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName );
void requestBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName );
void requestBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName );
void requestBlobDownload( const QString& blobId );
void requestBlobBySasUri( const QString& blobId, const QString& sasUri );
QByteArray requestParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName );
std::vector<SumoAsset> assets() const;
std::vector<SumoCase> cases() const;
std::vector<QString> ensembleNamesForCase( const SumoCaseId& caseId ) const;
std::vector<QString> vectorNames() const;
std::vector<QString> realizationIds() const;
std::vector<QString> blobIds() const;
std::vector<SumoRedirect> blobContents() const;
public slots:
void parseAssets( QNetworkReply* reply );
void parseEnsembleNames( QNetworkReply* reply, const SumoCaseId& caseId );
void parseCases( QNetworkReply* reply );
void parseVectorNames( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName );
void parseRealizationNumbers( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName );
void parseBlobId( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName, bool isParameters );
void requestFailed( const QAbstractOAuth::Error error );
void parquetDownloadComplete( const QString& blobId, const QByteArray&, const QString& url );
signals:
void fileDownloadFinished( const QString& fileId, const QString& filePath );
void casesFinished();
void wellsFinished();
void wellboresFinished( const QString& wellId );
void wellboreTrajectoryFinished( const QString& wellboreId );
void parquetDownloadFinished( const QByteArray& contents, const QString& url );
void ensembleNamesFinished();
void vectorNamesFinished();
void blobIdFinished();
void assetsFinished();
void realizationIdsFinished();
private:
void addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType );
QNetworkReply* makeDownloadRequest( const QString& url, const QString& token, const QString& contentType );
void requestParquetData( const QString& url, const QString& token );
// Run work on the transfer thread and wait for it. Pass progressText to show the standard progress dialog
// while waiting, worth doing for the transfers slow enough to be noticed and not for the small requests
// that would only make it flash.
void runOnTransferThreadBlocking( const std::function<void()>& work, const QString& progressText = {} );
// Run work on the transfer thread without waiting for it. The async data paths use this: the result is
// delivered by a callback rather than by returning, so the calling thread carries on immediately.
void runOnTransferThread( const std::function<void()>& work );
// Hand a call back to the thread the connector lives on, the one owning the user interface. Results of
// async work are delivered through this, so a caller never has its data handed to it on another thread.
void invokeOnConnectorThread( const std::function<void()>& work );
// Download one blob, calling onFinished with its contents. Call on the transfer thread, where onFinished
// is called as well. Empty contents mean the transfer failed.
void downloadBlobAsync( const QString& blobId, const std::function<void( const QByteArray& )>& onFinished );
// Abort a reply that has not finished in time, so an async chain reports a failure instead of hanging and
// leaving whoever waits for the data waiting forever.
static void abortIfNotFinishedWithin( QNetworkReply* reply, int timeoutMillis );
// The network manager belonging to the calling thread: the transfer thread manager when called from
// there, otherwise the one owned by RiaCloudConnector on the GUI thread.
QNetworkAccessManager* networkAccessManager();
// The token for a request issued from the transfer thread. token() reads objects owned by another thread.
QString transferToken() const;
static void waitForRepliesToFinish( const std::vector<QNetworkReply*>& replies );
// Issue and collect the two round trips a blob transfer needs. Called on the transfer thread.
std::map<QString, QByteArray> downloadBlobs( const std::vector<QString>& blobIds );
public slots:
void requestFailed( const QAbstractOAuth::Error error );
private:
// Call on the thread owning the authentication objects, before work is handed over.
void cacheTransferToken();
static QString constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken );
void wrapAndCallNetworkRequest( std::function<void()> requestCallable, const QMetaMethod& signalMethod );
QString sasUriFromReply( QNetworkReply* reply, const QString& blobId );
static QByteArray blobContentsFromReply( QNetworkReply* reply, const QString& sasUri );
static QByteArray replyBody( QNetworkReply* reply, const QString& url );
private:
std::function<QString()> m_serverUrlProvider;
std::vector<SumoAsset> m_assets;
std::vector<SumoCase> m_cases;
std::vector<QString> m_vectorNames;
std::vector<QString> m_realizationIds;
std::vector<SumoEnsemble> m_ensembleNames;
RiaSumoExplore m_explore;
RiaSumoSummary m_summary;
std::vector<QString> m_blobId;
// Transfers run on their own thread so the calling thread can wait without dispatching events. Waiting on
// a nested event loop on the GUI thread let the view update code re-enter a load that was still running,
// and the same grid property was downloaded twice. Authentication stays on the GUI thread: the OAuth flow
// opens a browser and its objects live there.
QThread* m_transferThread = nullptr;
QObject* m_transferContext = nullptr; // lives on the transfer thread
QNetworkAccessManager* m_transferNetworkAccessManager = nullptr; // created on the transfer thread
std::vector<SumoRedirect> m_redirectInfo;
// Written on the thread handing work over, read on the transfer thread running it.
mutable QMutex m_transferTokenMutex;
QString m_transferToken;
};
ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp
+8
View File
@@ -36,3 +36,11 @@ int RiaSumoDefines::requestTimeoutMillis()
{
return 10 * 1000;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
int RiaSumoDefines::asyncRequestTimeoutMillis()
{
return 5 * 60 * 1000;
}
ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h
+7
View File
@@ -22,6 +22,8 @@
#include <QString>
#include <cstddef>
using SumoAssetId = nonstd::ordered<QString, struct sumo_asset_tag>;
using SumoCaseId = nonstd::ordered<QString, struct sumo_case_tag>;
@@ -29,4 +31,9 @@ namespace RiaSumoDefines
{
QString tokenPath();
int requestTimeoutMillis();
// The timeout of a request nothing is waiting for. Only there so a request that never answers is eventually
// given up on, and generous because a summary vector that has not been aggregated yet is produced on demand
// by the request asking for it. Nothing is blocked while it runs, so waiting longer costs nothing.
int asyncRequestTimeoutMillis();
}; // namespace RiaSumoDefines
ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp
+168
View File
@@ -0,0 +1,168 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2024- Equinor ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#include "RiaSumoExplore.h"
#include "RiaLogging.h"
#include "RiaQStringFormatter.h"
#include "RiaSumoConnector.h"
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QUrl>
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
RiaSumoExplore::RiaSumoExplore( RiaSumoConnector& connector )
: m_connector( connector )
{
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<SumoAsset> RiaSumoExplore::assets()
{
const QString url = QString( "%1/assets" ).arg( m_connector.server() );
return parseAssets( m_connector.getBlocking( url, "Loading assets from Sumo" ) );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<SumoCase> RiaSumoExplore::cases( const QString& assetName )
{
const QString url = QString( "%1/cases?asset_name=%2" ).arg( m_connector.server() ).arg( QString( QUrl::toPercentEncoding( assetName ) ) );
return parseCases( m_connector.getBlocking( url, QString( "Loading the cases of %1 from Sumo" ).arg( assetName ) ) );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<QString> RiaSumoExplore::ensembleNames( const SumoCaseId& caseId )
{
const QString url = QString( "%1/cases/%2/ensembles" ).arg( m_connector.server() ).arg( caseId.get() );
return parseEnsembleNames( m_connector.getBlocking( url, "Loading ensembles from Sumo" ) );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<QString> RiaSumoExplore::realizationIds( const SumoCaseId& caseId, const QString& ensembleName )
{
const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName );
const QString url =
QString( "%1/cases/%2/ensembles/%3/realizations" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName );
return parseRealizationIds( m_connector.getBlocking( url, "Loading realizations from Sumo" ) );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<SumoAsset> RiaSumoExplore::parseAssets( const QByteArray& body )
{
std::vector<SumoAsset> assets;
QJsonDocument doc = QJsonDocument::fromJson( body );
QJsonArray jsonArray = doc.array();
// This json is an array of AssetInfo
for ( const QJsonValue& assetInfo : jsonArray )
{
QString assetName = assetInfo["name"].toString();
assets.push_back( SumoAsset{ SumoAssetId( "" ), "", assetName } );
}
for ( const auto& asset : assets )
{
RiaLogging::debug( std::format( "Asset: {}", asset.name ) );
}
return assets;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<SumoCase> RiaSumoExplore::parseCases( const QByteArray& body )
{
std::vector<SumoCase> cases;
QJsonDocument doc = QJsonDocument::fromJson( body );
QJsonArray jsonArray = doc.array();
for ( const QJsonValue& value : jsonArray )
{
QJsonObject caseObj = value.toObject();
QString id = caseObj["id"].toString();
QString kind = "";
QString name = caseObj["name"].toString();
cases.push_back( SumoCase{ SumoCaseId( id ), kind, name } );
}
RiaLogging::debug( std::format( "Case count : {}", cases.size() ) );
return cases;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<QString> RiaSumoExplore::parseEnsembleNames( const QByteArray& body )
{
std::vector<QString> ensembleNames;
QJsonDocument doc = QJsonDocument::fromJson( body );
QJsonArray jsonArray = doc.array();
for ( const QJsonValue& value : jsonArray )
{
QJsonObject ensembleObj = value.toObject();
ensembleNames.push_back( ensembleObj["name"].toString() );
}
RiaLogging::debug( std::format( "Ensemble count : {}", ensembleNames.size() ) );
return ensembleNames;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<QString> RiaSumoExplore::parseRealizationIds( const QByteArray& body )
{
std::vector<QString> realizationIds;
QJsonDocument doc = QJsonDocument::fromJson( body );
QJsonArray jsonArray = doc.array();
for ( const QJsonValue& value : jsonArray )
{
realizationIds.push_back( QString::number( value.toInt() ) );
}
return realizationIds;
}
ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.h
+70
View File
@@ -0,0 +1,70 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2024- Equinor ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "RiaSumoDefines.h"
#include <QByteArray>
#include <QString>
#include <vector>
class RiaSumoConnector;
struct SumoAsset
{
SumoAssetId assetId;
QString kind;
QString name;
};
struct SumoCase
{
SumoCaseId caseId;
QString kind;
QString name;
};
//==================================================================================================
/// Finding your way around what Sumo holds: the assets available, the cases of an asset, the ensembles
/// of a case and the realizations of an ensemble. Requests are made through RiaSumoConnector, which owns
/// the connection and does the transfers, and every call returns its result rather than leaving it in
/// shared state.
//==================================================================================================
class RiaSumoExplore
{
public:
explicit RiaSumoExplore( RiaSumoConnector& connector );
std::vector<SumoAsset> assets();
std::vector<SumoCase> cases( const QString& assetName );
std::vector<QString> ensembleNames( const SumoCaseId& caseId );
std::vector<QString> realizationIds( const SumoCaseId& caseId, const QString& ensembleName );
private:
static std::vector<SumoAsset> parseAssets( const QByteArray& body );
static std::vector<SumoCase> parseCases( const QByteArray& body );
static std::vector<QString> parseEnsembleNames( const QByteArray& body );
static std::vector<QString> parseRealizationIds( const QByteArray& body );
private:
RiaSumoConnector& m_connector;
};
ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp
+370
View File
@@ -0,0 +1,370 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2024- Equinor ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#include "RiaSumoSummary.h"
#include "RiaCloudDefines.h"
#include "RiaLogging.h"
#include "RiaSumoConnector.h"
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QUrl>
#include <algorithm>
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
RiaSumoSummary::RiaSumoSummary( RiaSumoConnector& connector )
: m_connector( connector )
{
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<QString> RiaSumoSummary::vectorNames( const SumoCaseId& caseId, const QString& ensembleName )
{
const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName );
const QString url =
QString( "%1/cases/%2/ensembles/%3/vector_list" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName );
return parseVectorNames( m_connector.getBlocking( url ) );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QByteArray RiaSumoSummary::vectorData( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName )
{
const auto contentsByVectorName = vectorData( caseId, ensembleName, std::vector<QString>{ vectorName } );
if ( auto it = contentsByVectorName.find( vectorName ); it != contentsByVectorName.end() ) return it->second;
return {};
}
//--------------------------------------------------------------------------------------------------
/// Fetch several summary vectors at the same time. The blob id requests are issued together and waited
/// for as a group, and so are the transfers, turning 2N sequential round trips into 2 batched ones.
///
/// This matters more than the round trip count alone: a vector that has not been aggregated yet is
/// produced on demand by the request that asks for it, so fetching serially costs the sum of those
/// aggregations while fetching together costs roughly the slowest one.
//--------------------------------------------------------------------------------------------------
std::map<QString, QByteArray>
RiaSumoSummary::vectorData( const SumoCaseId& caseId, const QString& ensembleName, const std::vector<QString>& vectorNames )
{
std::map<QString, QByteArray> contentsByVectorName;
if ( vectorNames.empty() ) return contentsByVectorName;
// Drop duplicates, so a vector is never requested twice in one batch.
std::vector<QString> namesToFetch;
for ( const auto& vectorName : vectorNames )
{
if ( vectorName.isEmpty() ) continue;
if ( std::ranges::find( namesToFetch, vectorName ) != namesToFetch.end() ) continue;
namesToFetch.push_back( vectorName );
}
if ( namesToFetch.empty() ) return contentsByVectorName;
m_connector.runOnTransferThreadBlocking(
[&]()
{
// Phase 1: resolve all blob ids concurrently.
std::vector<QNetworkReply*> blobIdReplies;
for ( const auto& vectorName : namesToFetch )
{
blobIdReplies.push_back( makeVectorBlobIdRequest( caseId, ensembleName, vectorName ) );
}
RiaSumoConnector::waitForRepliesToFinish( blobIdReplies );
std::vector<QString> blobIds;
for ( size_t i = 0; i < blobIdReplies.size(); i++ )
{
blobIds.push_back( blobIdFromReply( blobIdReplies[i], namesToFetch[i] ) );
}
// Phase 2: download all resolved blobs as one group.
std::vector<QString> blobIdsToDownload;
for ( const auto& blobId : blobIds )
{
if ( !blobId.isEmpty() ) blobIdsToDownload.push_back( blobId );
}
const auto contentsByBlobId = m_connector.downloadBlobs( blobIdsToDownload );
// Anything missing failed; the caller falls back to fetching it on its own later.
for ( size_t i = 0; i < blobIds.size(); i++ )
{
if ( blobIds[i].isEmpty() ) continue;
if ( auto it = contentsByBlobId.find( blobIds[i] ); it != contentsByBlobId.end() )
{
contentsByVectorName[namesToFetch[i]] = it->second;
}
}
},
QString( "Loading %1 summary vector(s) from Sumo" ).arg( namesToFetch.size() ) );
return contentsByVectorName;
}
//--------------------------------------------------------------------------------------------------
/// Fetch several summary vectors without waiting for any of them. Every vector is requested at once and
/// onVectorReady is called for each one as it arrives, on the thread the connector lives on, so a caller can
/// show each vector the moment it is there instead of when the slowest one is.
///
/// Empty contents mean that vector failed. The callback is called exactly once per requested vector, so a
/// caller tracking what is still on its way can rely on all of them being accounted for.
//--------------------------------------------------------------------------------------------------
void RiaSumoSummary::vectorDataAsync( const SumoCaseId& caseId,
const QString& ensembleName,
const std::vector<QString>& vectorNames,
const std::function<void( const QString&, const QByteArray& )>& onVectorReady )
{
if ( vectorNames.empty() || !onVectorReady ) return;
m_connector.runOnTransferThread(
[this, caseId, ensembleName, vectorNames, onVectorReady]()
{
for ( const auto& vectorName : vectorNames )
{
auto deliver = [this, onVectorReady, vectorName]( const QByteArray& contents )
{
m_connector.invokeOnConnectorThread( [onVectorReady, vectorName, contents]() { onVectorReady( vectorName, contents ); } );
};
auto blobIdReply = makeVectorBlobIdRequest( caseId, ensembleName, vectorName );
if ( !blobIdReply )
{
deliver( {} );
continue;
}
// A vector that has not been aggregated yet is produced on demand by this request, which can
// take a good while. Nothing is waiting on it, so it is given room to finish.
RiaSumoConnector::abortIfNotFinishedWithin( blobIdReply, RiaSumoDefines::asyncRequestTimeoutMillis() );
QObject::connect( blobIdReply,
&QNetworkReply::finished,
blobIdReply,
[this, blobIdReply, vectorName, deliver]()
{
const QString blobId = blobIdFromReply( blobIdReply, vectorName );
if ( blobId.isEmpty() )
{
deliver( {} );
return;
}
m_connector.downloadBlobAsync( blobId, deliver );
} );
}
} );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QByteArray RiaSumoSummary::parameterData( const SumoCaseId& caseId, const QString& ensembleName )
{
const QString blobId = parameterBlobId( caseId, ensembleName );
if ( blobId.isEmpty() ) return {};
return m_connector.downloadBlobBlocking( blobId );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QString RiaSumoSummary::vectorBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName )
{
const QString url = vectorBlobIdUrl( caseId, ensembleName, vectorName );
return logBlobId( RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( url ) ), vectorName );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QString RiaSumoSummary::vectorBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) const
{
const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName );
const QString encodedVectorName = QUrl::toPercentEncoding( vectorName );
return QString( "%1/cases/%2/ensembles/%3/vectors/%4/blob_id" )
.arg( m_connector.server() )
.arg( caseId.get() )
.arg( encodedEnsembleName )
.arg( encodedVectorName );
}
//--------------------------------------------------------------------------------------------------
/// Issue the blob id request for one vector. The reply is returned unfinished, so the caller decides how
/// to wait for it: one at a time, or several at once when batching.
//--------------------------------------------------------------------------------------------------
QNetworkReply* RiaSumoSummary::makeVectorBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName )
{
QNetworkRequest networkRequest;
networkRequest.setUrl( QUrl( vectorBlobIdUrl( caseId, ensembleName, vectorName ) ) );
m_connector.addStandardHeader( networkRequest, m_connector.transferToken(), RiaCloudDefines::contentTypeJson() );
return m_connector.networkAccessManager()->get( networkRequest );
}
//--------------------------------------------------------------------------------------------------
/// Read the blob id off a finished blob id reply. The reply is consumed and scheduled for deletion.
//--------------------------------------------------------------------------------------------------
QString RiaSumoSummary::blobIdFromReply( QNetworkReply* reply, const QString& vectorName )
{
if ( !reply ) return {};
const bool failed = !reply->isFinished() || reply->error() != QNetworkReply::NoError;
QByteArray body = failed ? QByteArray() : reply->readAll();
if ( failed )
{
RiaLogging::error(
std::format( "Request blob ID failed for vector '{}': {}", vectorName.toStdString(), reply->errorString().toStdString() ) );
}
reply->deleteLater();
return logBlobId( RiaSumoConnector::blobIdFromBody( body ), vectorName );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QString RiaSumoSummary::logBlobId( const QString& blobId, const QString& vectorName )
{
if ( !blobId.isEmpty() )
{
RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", vectorName.toStdString(), blobId.toStdString() ) );
}
return blobId;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QString RiaSumoSummary::parameterBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName ) const
{
const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName );
return QString( "%1/cases/%2/ensembles/%3/parameters/blob_id" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName );
}
//--------------------------------------------------------------------------------------------------
/// Issue the blob id request for the ensemble parameters. The reply is returned unfinished, so the caller
/// decides how to wait for it.
//--------------------------------------------------------------------------------------------------
QNetworkReply* RiaSumoSummary::makeParameterBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName )
{
QNetworkRequest networkRequest;
networkRequest.setUrl( QUrl( parameterBlobIdUrl( caseId, ensembleName ) ) );
m_connector.addStandardHeader( networkRequest, m_connector.transferToken(), RiaCloudDefines::contentTypeJson() );
return m_connector.networkAccessManager()->get( networkRequest );
}
//--------------------------------------------------------------------------------------------------
/// Fetch the ensemble parameters without waiting, calling onParametersReady on the thread the connector
/// lives on. Empty contents mean the request failed, and the callback is called exactly once.
//--------------------------------------------------------------------------------------------------
void RiaSumoSummary::parameterDataAsync( const SumoCaseId& caseId,
const QString& ensembleName,
const std::function<void( const QByteArray& )>& onParametersReady )
{
if ( !onParametersReady ) return;
m_connector.runOnTransferThread(
[this, caseId, ensembleName, onParametersReady]()
{
auto deliver = [this, onParametersReady]( const QByteArray& contents )
{ m_connector.invokeOnConnectorThread( [onParametersReady, contents]() { onParametersReady( contents ); } ); };
auto blobIdReply = makeParameterBlobIdRequest( caseId, ensembleName );
if ( !blobIdReply )
{
deliver( {} );
return;
}
RiaSumoConnector::abortIfNotFinishedWithin( blobIdReply, RiaSumoDefines::asyncRequestTimeoutMillis() );
QObject::connect( blobIdReply,
&QNetworkReply::finished,
blobIdReply,
[this, blobIdReply, deliver]()
{
const QString blobId = blobIdFromReply( blobIdReply, "parameters" );
if ( blobId.isEmpty() )
{
deliver( {} );
return;
}
m_connector.downloadBlobAsync( blobId, deliver );
} );
} );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QString RiaSumoSummary::parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName )
{
const QString blobId = RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( parameterBlobIdUrl( caseId, ensembleName ) ) );
if ( !blobId.isEmpty() )
{
RiaLogging::debug( std::format( "Received blob ID for parameters: {}", blobId.toStdString() ) );
}
return blobId;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<QString> RiaSumoSummary::parseVectorNames( const QByteArray& body )
{
std::vector<QString> vectorNames;
QJsonDocument doc = QJsonDocument::fromJson( body );
QJsonArray jsonArray = doc.array();
for ( const QJsonValue& value : jsonArray )
{
QJsonObject vectorObj = value.toObject();
vectorNames.push_back( vectorObj["name"].toString() );
}
return vectorNames;
}
ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h
+85
View File
@@ -0,0 +1,85 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2024- Equinor ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "RiaSumoDefines.h"
#include <QByteArray>
#include <QString>
#include <functional>
#include <map>
#include <vector>
class RiaSumoConnector;
class QNetworkReply;
//==================================================================================================
/// The summary data of a Sumo case: the vectors an ensemble has, their values, and the ensemble
/// parameters. Requests are made through RiaSumoConnector, which owns the connection and does the
/// transfers, and every call returns its result rather than leaving it in shared state.
//==================================================================================================
class RiaSumoSummary
{
public:
explicit RiaSumoSummary( RiaSumoConnector& connector );
std::vector<QString> vectorNames( const SumoCaseId& caseId, const QString& ensembleName );
// The values of one summary vector, for all realizations, as a parquet blob.
QByteArray vectorData( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName );
// The same for several vectors at once, returned by vector name. The blob id requests are issued as one
// concurrent group and so are the transfers, which matters because a vector that has not been aggregated
// yet is produced on demand by the request asking for it.
std::map<QString, QByteArray> vectorData( const SumoCaseId& caseId, const QString& ensembleName, const std::vector<QString>& vectorNames );
// The same again, but without waiting: all vectors are requested at once and onVectorReady is called for
// each as it arrives, on the thread the connector lives on. Empty contents mean that vector failed, and
// the callback is called exactly once per requested vector.
void vectorDataAsync( const SumoCaseId& caseId,
const QString& ensembleName,
const std::vector<QString>& vectorNames,
const std::function<void( const QString&, const QByteArray& )>& onVectorReady );
// The ensemble parameters, as a parquet blob.
QByteArray parameterData( const SumoCaseId& caseId, const QString& ensembleName );
// The same without waiting. Like the vectors, the parameters are aggregated on demand by the service, so
// the first request for them can take a while and is not something to hold the user interface for.
void parameterDataAsync( const SumoCaseId& caseId,
const QString& ensembleName,
const std::function<void( const QByteArray& )>& onParametersReady );
QString vectorBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName );
QString parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName );
private:
QString vectorBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) const;
QString parameterBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName ) const;
QNetworkReply* makeParameterBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName );
QNetworkReply* makeVectorBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName );
static QString blobIdFromReply( QNetworkReply* reply, const QString& vectorName );
static QString logBlobId( const QString& blobId, const QString& vectorName );
static std::vector<QString> parseVectorNames( const QByteArray& body );
private:
RiaSumoConnector& m_connector;
};
ApplicationLibCode/Application/Tools/RiaLogging.cpp
+11
View File
@@ -240,6 +240,17 @@ void RiaLogging::appendLoggerInstance( std::unique_ptr<RiaLogger> loggerInstance
sm_logger.push_back( std::move( loggerInstance ) );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaLogging::flushPendingMessages()
{
for ( const auto& logger : sm_logger )
{
logger->flushPendingMessages();
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
ApplicationLibCode/Application/Tools/RiaLogging.h
+8
View File
@@ -52,6 +52,10 @@ public:
virtual void warning( const char* message ) = 0;
virtual void info( const char* message ) = 0;
virtual void debug( const char* message ) = 0;
// Deliver messages a logger has accepted but not yet written out. A logger that has to hand messages from
// a worker thread over to another thread can otherwise let them arrive after messages logged later.
virtual void flushPendingMessages() {}
};
//==================================================================================================
@@ -73,6 +77,10 @@ public:
static void info( std::string_view message, std::string_view logKeyword = "" );
static void debug( std::string_view message, std::string_view logKeyword = "" );
// Write out anything the loggers are holding, so messages logged from a worker thread appear before the
// messages the waiting thread logs once the worker is done.
static void flushPendingMessages();
static std::chrono::time_point<std::chrono::high_resolution_clock> currentTime();
static void logElapsedTime( std::string_view message, const std::chrono::time_point<std::chrono::high_resolution_clock>& startTime );
ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp
+23 -25
View File
@@ -93,9 +93,11 @@ SimpleDialog::SimpleDialog( QWidget* parent )
//--------------------------------------------------------------------------------------------------
SimpleDialog::~SimpleDialog()
{
// The connector belongs to RiaApplication and is shared with everything else reading from Sumo, so it
// must outlive this dialog. Only the connection made in createConnection is ours to undo.
if ( m_sumoConnector )
{
m_sumoConnector->deleteLater();
disconnect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady );
}
}
@@ -105,7 +107,10 @@ SimpleDialog::~SimpleDialog()
void SimpleDialog::createConnection()
{
m_sumoConnector = RiaApplication::instance()->makeSumoConnector();
connect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady );
// Authenticating more than once would otherwise leave one connection per attempt, and onTokenReady
// would be called once for each of them.
connect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady, Qt::UniqueConnection );
}
//--------------------------------------------------------------------------------------------------
@@ -124,10 +129,9 @@ void SimpleDialog::onAssetsClicked()
{
if ( !isTokenValid() ) return;
m_sumoConnector->requestAssetsBlocking();
m_sumoConnector->assets();
const auto assets = m_sumoConnector->explore().assets();
label->setText( "Requesting fields (see log for response" );
label->setText( QString( "Received %1 assets" ).arg( assets.size() ) );
}
//--------------------------------------------------------------------------------------------------
@@ -137,10 +141,10 @@ void SimpleDialog::onCasesClicked()
{
if ( !isTokenValid() ) return;
QString fieldName = "Drogon";
m_sumoConnector->requestCasesForField( fieldName );
QString fieldName = "Drogon";
const auto cases = m_sumoConnector->explore().cases( fieldName );
label->setText( "Requesting cases (see log for response" );
label->setText( QString( "Received %1 cases" ).arg( cases.size() ) );
}
//--------------------------------------------------------------------------------------------------
@@ -153,9 +157,9 @@ void SimpleDialog::onVectorNamesClicked()
SumoCaseId caseId( "5b783aab-ce10-4b78-b129-baf8d8ce4baa" );
QString iteration = "iter-0";
m_sumoConnector->requestVectorNamesForEnsemble( caseId, iteration );
const auto vectorNames = m_sumoConnector->summary().vectorNames( caseId, iteration );
label->setText( "Requesting vector names (see log for response" );
label->setText( QString( "Received %1 vector names" ).arg( vectorNames.size() ) );
}
//--------------------------------------------------------------------------------------------------
@@ -169,9 +173,9 @@ void SimpleDialog::onFindBlobIdClicked()
QString iteration = "iter-0";
QString vectorName = "FOPT";
m_sumoConnector->requestBlobIdForEnsemble( caseId, iteration, vectorName );
m_blobId = m_sumoConnector->summary().vectorBlobId( caseId, iteration, vectorName );
label->setText( "Requesting blob ID for vector name (see log for response" );
label->setText( m_blobId.isEmpty() ? QString( "No blob ID received" ) : QString( "Blob ID: %1" ).arg( m_blobId ) );
}
//--------------------------------------------------------------------------------------------------
@@ -181,16 +185,16 @@ void SimpleDialog::onParquetClicked()
{
if ( !isTokenValid() ) return;
if ( m_sumoConnector->blobIds().empty() )
if ( m_blobId.isEmpty() )
{
onFindBlobIdClicked();
}
if ( !m_sumoConnector->blobIds().empty() )
if ( !m_blobId.isEmpty() )
{
m_sumoConnector->requestBlobDownload( m_sumoConnector->blobIds().back() );
m_blobContents = m_sumoConnector->downloadBlobBlocking( m_blobId );
label->setText( "Requesting blob ID for vector name (see log for response" );
label->setText( QString( "Downloaded blob, %1 bytes" ).arg( m_blobContents.size() ) );
}
}
@@ -199,14 +203,10 @@ void SimpleDialog::onParquetClicked()
//--------------------------------------------------------------------------------------------------
void SimpleDialog::onShowContentParquetClicked()
{
if ( m_sumoConnector->blobContents().empty() ) return;
auto blob = m_sumoConnector->blobContents().back();
auto content = blob.contents;
if ( m_blobContents.isEmpty() ) return;
// TODO: show content using parquet reader
auto tableText = RifArrowTools::readFirstRowsOfTable( content );
auto tableText = RifArrowTools::readFirstRowsOfTable( m_blobContents );
RiaLogging::info( tableText.toStdString() );
}
@@ -220,9 +220,7 @@ void SimpleDialog::onRealizationsClicked()
SumoCaseId caseId( "485041ce-ad72-48a3-ac8c-484c0ed95cf8" );
QString iteration = "iter-0";
m_sumoConnector->requestRealizationIdsForEnsembleBlocking( caseId, iteration );
auto ids = m_sumoConnector->realizationIds();
auto ids = m_sumoConnector->explore().realizationIds( caseId, iteration );
for ( const auto& id : ids )
{
RiaLogging::info( id.toStdString() );
ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h
+5
View File
@@ -67,6 +67,11 @@ private:
QPushButton* realizationIdsButton;
QPointer<RiaSumoConnector> m_sumoConnector;
// The most recently resolved blob id, and the blob downloaded from it, kept here so the buttons that
// follow have something to work on.
QString m_blobId;
QByteArray m_blobContents;
};
//==================================================================================================
ApplicationLibCode/Commands/SummaryPlotCommands/RicSummaryPlotEditorUi.cpp
+32
View File
@@ -44,6 +44,7 @@
#include "RimSummaryEnsemble.h"
#include "RimSummaryMultiPlot.h"
#include "RimSummaryPlot.h"
#include "Sumo/RimSummaryEnsembleSumo.h"
#include "RiuPlotMainWindow.h"
#include "RiuPlotMainWindowTools.h"
@@ -456,6 +457,37 @@ void RicSummaryPlotEditorUi::updatePreviewCurvesFromCurveDefinitions( const std:
m_previewPlot->zoomAll();
m_previewPlot->updateConnectedEditors();
m_previewPlot->summaryCurveCollection()->updateConnectedEditors();
listenForSummaryDataLoaded();
}
//--------------------------------------------------------------------------------------------------
/// Follow the ensembles the preview plot now reads from. Connecting twice to the same one is harmless, and the
/// set changes as the selection does, so this is redone whenever the preview curves are rebuilt.
//--------------------------------------------------------------------------------------------------
void RicSummaryPlotEditorUi::listenForSummaryDataLoaded()
{
for ( const auto& [ensemble, addresses] : m_previewPlot->summaryAddressesByEnsemble() )
{
// Only a source that loads without waiting has anything to announce.
if ( auto sumoEnsemble = dynamic_cast<RimSummaryEnsembleSumo*>( ensemble ) )
{
sumoEnsemble->summaryDataLoaded.connect( this, &RicSummaryPlotEditorUi::onSummaryDataLoaded );
}
}
}
//--------------------------------------------------------------------------------------------------
/// Data the preview plot was waiting for has arrived. The axes are fitted as well, as the plot was drawn while
/// it had nothing to fit to.
//--------------------------------------------------------------------------------------------------
void RicSummaryPlotEditorUi::onSummaryDataLoaded( const caf::SignalEmitter* emitter )
{
if ( !m_previewPlot ) return;
m_previewPlot->loadDataAndUpdate();
m_previewPlot->zoomAll();
m_previewPlot->scheduleReplotIfVisible();
}
//--------------------------------------------------------------------------------------------------
ApplicationLibCode/Commands/SummaryPlotCommands/RicSummaryPlotEditorUi.h
+5
View File
@@ -80,6 +80,11 @@ private:
void defineEditorAttribute( const caf::PdmFieldHandle* field, QString uiConfigName, caf::PdmUiEditorAttribute* attribute ) override;
void syncPreviewCurvesFromUiSelection();
// The preview plot lives outside the project, so a source loading without waiting cannot reach it.
void listenForSummaryDataLoaded();
void onSummaryDataLoaded( const caf::SignalEmitter* emitter );
void updatePreviewCurvesFromCurveDefinitions( const std::set<RiaSummaryCurveDefinition>& allCurveDefsToDisplay,
const std::set<RiaSummaryCurveDefinition>& curveDefsToAdd,
const std::set<RimSummaryCurve*>& curvesToDelete,
ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp
+76 -27
View File
@@ -167,14 +167,21 @@ void RimCloudDataSourceCollection::fieldChangedByUi( const caf::PdmFieldHandle*
if ( changedField == &m_sumoFieldName )
{
// What was picked below belonged to the asset that was just left, both the selection and the options
// it was chosen from. Forget the cached answers as well, or the case list would keep offering the
// cases of the previous asset. The editors are refreshed by the caller, which updates them as soon
// as this returns, so asking for that here would only fetch everything a second time.
m_sumoCaseId = "";
m_sumoEnsembleNames.v().clear();
m_sumoEnsembleNames.setValue( {} );
m_sumoConnector->requestCasesForFieldBlocking( m_sumoFieldName );
clearCachedCases();
clearCachedEnsembleNames();
}
else if ( changedField == &m_sumoCaseId )
{
m_sumoEnsembleNames.v().clear();
m_sumoEnsembleNames.setValue( {} );
clearCachedEnsembleNames();
}
if ( changedField == &m_addEnsembles )
{
@@ -214,12 +221,7 @@ QList<caf::PdmOptionItemInfo> RimCloudDataSourceCollection::calculateValueOption
QList<caf::PdmOptionItemInfo> options;
if ( fieldNeedingOptions == &m_sumoFieldName )
{
if ( m_sumoConnector->assets().empty() )
{
m_sumoConnector->requestAssetsBlocking();
}
for ( const auto& asset : m_sumoConnector->assets() )
for ( const auto& asset : cachedAssets() )
{
if ( m_sumoFieldName().isEmpty() )
{
@@ -231,24 +233,14 @@ QList<caf::PdmOptionItemInfo> RimCloudDataSourceCollection::calculateValueOption
}
else if ( fieldNeedingOptions == &m_sumoCaseId && !m_sumoFieldName().isEmpty() )
{
if ( m_sumoConnector->cases().empty() )
{
m_sumoConnector->requestCasesForFieldBlocking( m_sumoFieldName );
}
for ( const auto& sumoCase : m_sumoConnector->cases() )
for ( const auto& sumoCase : cachedCases( m_sumoFieldName ) )
{
options.push_back( { sumoCase.name, sumoCase.caseId.get() } );
}
}
else if ( fieldNeedingOptions == &m_sumoEnsembleNames && !m_sumoCaseId().isEmpty() )
{
if ( m_sumoConnector->ensembleNamesForCase( SumoCaseId( m_sumoCaseId ) ).empty() )
{
m_sumoConnector->requestEnsembleByCasesIdBlocking( SumoCaseId( m_sumoCaseId ) );
}
for ( const auto& name : m_sumoConnector->ensembleNamesForCase( SumoCaseId( m_sumoCaseId ) ) )
for ( const auto& name : cachedEnsembleNames( SumoCaseId( m_sumoCaseId ) ) )
{
options.push_back( { name, name } );
}
@@ -257,6 +249,66 @@ QList<caf::PdmOptionItemInfo> RimCloudDataSourceCollection::calculateValueOption
return options;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
const std::vector<SumoAsset>& RimCloudDataSourceCollection::cachedAssets()
{
if ( m_assets.empty() && m_sumoConnector )
{
m_assets = m_sumoConnector->explore().assets();
}
return m_assets;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
const std::vector<SumoCase>& RimCloudDataSourceCollection::cachedCases( const QString& assetName )
{
if ( m_casesAssetName != assetName && m_sumoConnector )
{
m_cases = m_sumoConnector->explore().cases( assetName );
m_casesAssetName = assetName;
}
return m_cases;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
const std::vector<QString>& RimCloudDataSourceCollection::cachedEnsembleNames( const SumoCaseId& caseId )
{
if ( m_ensembleNamesCaseId != caseId.get() && m_sumoConnector )
{
m_ensembleNames = m_sumoConnector->explore().ensembleNames( caseId );
m_ensembleNamesCaseId = caseId.get();
}
return m_ensembleNames;
}
//--------------------------------------------------------------------------------------------------
/// Forget the cases Sumo answered with, so the next request for them asks again. Called when the asset they
/// belong to is left behind.
//--------------------------------------------------------------------------------------------------
void RimCloudDataSourceCollection::clearCachedCases()
{
m_casesAssetName.clear();
m_cases.clear();
}
//--------------------------------------------------------------------------------------------------
/// The same for the ensemble names, which belong to a case.
//--------------------------------------------------------------------------------------------------
void RimCloudDataSourceCollection::clearCachedEnsembleNames()
{
m_ensembleNamesCaseId.clear();
m_ensembleNames.clear();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
@@ -402,7 +454,7 @@ std::vector<RimSumoDataSource*> RimCloudDataSourceCollection::addDataSources()
}
QString caseName;
for ( const auto& sumoCase : m_sumoConnector->cases() )
for ( const auto& sumoCase : cachedCases( m_sumoFieldName ) )
{
if ( sumoCase.caseId == sumoCaseId )
{
@@ -411,11 +463,8 @@ std::vector<RimSumoDataSource*> RimCloudDataSourceCollection::addDataSources()
}
}
m_sumoConnector->requestRealizationIdsForEnsembleBlocking( sumoCaseId, ensembleName );
m_sumoConnector->requestVectorNamesForEnsembleBlocking( sumoCaseId, ensembleName );
auto availableRealizationIds = m_sumoConnector->realizationIds();
auto vectorNames = m_sumoConnector->vectorNames();
const auto availableRealizationIds = m_sumoConnector->explore().realizationIds( sumoCaseId, ensembleName );
const auto vectorNames = m_sumoConnector->summary().vectorNames( sumoCaseId, ensembleName );
auto dataSource = new RimSumoDataSource();
dataSource->setCaseId( sumoCaseId );
ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h
+18
View File
@@ -58,6 +58,16 @@ private:
static bool isCloudApiServerAvailable();
// The option lists are rebuilt every time the property editor refreshes, so what Sumo answered is kept
// here and only asked for again when the selection it belongs to changes.
const std::vector<SumoAsset>& cachedAssets();
const std::vector<SumoCase>& cachedCases( const QString& assetName );
const std::vector<QString>& cachedEnsembleNames( const SumoCaseId& caseId );
void clearCachedCases();
void clearCachedEnsembleNames();
private:
caf::PdmField<bool> m_authenticate;
caf::PdmField<QString> m_sumoFieldName;
@@ -73,4 +83,12 @@ private:
caf::PdmField<bool> m_restartServer;
QPointer<RiaSumoConnector> m_sumoConnector;
std::vector<SumoAsset> m_assets;
QString m_casesAssetName;
std::vector<SumoCase> m_cases;
QString m_ensembleNamesCaseId;
std::vector<QString> m_ensembleNames;
};
ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp
+4
View File
@@ -857,6 +857,10 @@ void RimMultiPlot::updatePlots()
{
if ( m_showWindow )
{
// The plots are loaded one after the other, so a source loading data remotely would make its requests
// one plot at a time. Give it the chance to load everything this window needs in one go first.
prefetchPlotData();
for ( RimPlot* plot : plots() )
{
plot->loadDataAndUpdate();
ApplicationLibCode/ProjectDataModel/RimMultiPlot.h
+5
View File
@@ -142,6 +142,11 @@ protected:
virtual void onPlotAdditionOrRemoval();
// Called before the plots are loaded, giving a window the chance to load what all of its plots need at
// once instead of leaving each plot to request its own. Data is still loaded on demand, so this is an
// optimization only: not overriding it changes nothing but speed.
virtual void prefetchPlotData() {}
bool isMouseCursorInsidePlot();
private:
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h
+9
View File
@@ -83,6 +83,15 @@ public:
virtual std::set<RifEclipseSummaryAddress> ensembleSummaryAddresses() const;
virtual std::set<time_t> ensembleTimeSteps() const;
// Hint that these addresses are about to be read, given before the curves pull their values. Sources that
// fetch data remotely can use it to load everything in one go instead of one blocking request per address.
// Data is still loaded on demand, so this is an optimization only: not calling it changes nothing but speed.
virtual void prefetchSummaryData( const std::vector<RifEclipseSummaryAddress>& resultAddresses ) {}
// Whether any of these addresses is being loaded right now, for sources that load without waiting. A plot
// asks about the addresses of its own curves, so it can say it is still waiting for data.
virtual bool isSummaryDataPending( const std::vector<RifEclipseSummaryAddress>& resultAddresses ) const { return false; }
void setEnsembleId( int ensembleId );
int ensembleId() const;
bool hasEnsembleParameters() const;
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp
+26
View File
@@ -1588,6 +1588,32 @@ void RimSummaryMultiPlot::onPlotAdditionOrRemoval()
RimMultiPlot::onPlotAdditionOrRemoval();
}
//--------------------------------------------------------------------------------------------------
/// Gather what every plot in the window is about to read and hand it to each ensemble as one set. A plot
/// prefetches for itself as well, but the plots are loaded one at a time, so doing it here is what lets a
/// remote source load the whole window in one request group rather than one per plot.
//--------------------------------------------------------------------------------------------------
void RimSummaryMultiPlot::prefetchPlotData()
{
std::map<RimSummaryEnsemble*, std::vector<RifEclipseSummaryAddress>> addressesByEnsemble;
for ( RimSummaryPlot* plot : summaryPlots() )
{
if ( !plot ) continue;
for ( const auto& [ensemble, addresses] : plot->summaryAddressesByEnsemble() )
{
auto& allAddresses = addressesByEnsemble[ensemble];
allAddresses.insert( allAddresses.end(), addresses.begin(), addresses.end() );
}
}
for ( const auto& [ensemble, addresses] : addressesByEnsemble )
{
ensemble->prefetchSummaryData( addresses );
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h
+2
View File
@@ -123,6 +123,8 @@ protected:
void onPlotAdditionOrRemoval() override;
void prefetchPlotData() override;
private:
void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override;
void defineEditorAttribute( const caf::PdmFieldHandle* field, QString uiConfigName, caf::PdmUiEditorAttribute* attribute ) override;
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp
+109
View File
@@ -63,6 +63,8 @@
#include "SummaryPlotCommands/RicSummaryPlotEditorUi.h"
#include "Tools/RimPlotAxisTools.h"
#include "RiuAbstractOverlayContentFrame.h"
#include "RiuDraggableOverlayFrame.h"
#include "RiuPlotAxis.h"
#include "RiuPlotMainWindow.h"
#include "RiuPlotMainWindowTools.h"
@@ -1805,6 +1807,109 @@ void RimSummaryPlot::defineUiTreeOrdering( caf::PdmUiTreeOrdering& uiTreeOrderin
uiTreeOrdering.skipRemainingChildren( true );
}
//--------------------------------------------------------------------------------------------------
/// The addresses this plot's curves are about to read, grouped by the ensemble owning them. Used to tell a
/// source what is coming before any of it is read, see prefetchSummaryData().
//--------------------------------------------------------------------------------------------------
std::map<RimSummaryEnsemble*, std::vector<RifEclipseSummaryAddress>> RimSummaryPlot::summaryAddressesByEnsemble() const
{
std::map<RimSummaryEnsemble*, std::vector<RifEclipseSummaryAddress>> addressesByEnsemble;
auto addAddress = []( auto& addressesByEnsemble, RimSummaryEnsemble* ensemble, const RifEclipseSummaryAddress& address )
{
if ( !ensemble || !address.isValid() ) return;
addressesByEnsemble[ensemble].push_back( address );
};
if ( m_summaryCurveCollection )
{
for ( RimSummaryCurve* curve : m_summaryCurveCollection->curves() )
{
if ( !curve ) continue;
if ( auto summaryCase = curve->summaryCaseY() )
{
addAddress( addressesByEnsemble, summaryCase->firstAncestorOrThisOfType<RimSummaryEnsemble>(), curve->summaryAddressY() );
}
if ( auto summaryCase = curve->summaryCaseX() )
{
addAddress( addressesByEnsemble, summaryCase->firstAncestorOrThisOfType<RimSummaryEnsemble>(), curve->summaryAddressX() );
}
}
}
for ( RimEnsembleCurveSet* curveSet : m_ensembleCurveSetCollection->curveSets() )
{
if ( !curveSet ) continue;
addAddress( addressesByEnsemble, curveSet->summaryEnsemble(), curveSet->summaryAddressY() );
}
return addressesByEnsemble;
}
//--------------------------------------------------------------------------------------------------
/// Tell each ensemble in the plot which addresses its curves are about to read, before any of them read
/// one. Curves pull their values one at a time, so a source loading data remotely would otherwise make one
/// blocking request per curve; given the whole set up front it can load them together.
///
/// This is a hint only. Every curve still loads its own data, and an ensemble that does not need the hint
/// ignores it. A plot in a plot window is usually covered by the window prefetching for all its plots at
/// once, in which case there is nothing left for this to do.
//--------------------------------------------------------------------------------------------------
void RimSummaryPlot::prefetchSummaryData()
{
for ( const auto& [ensemble, addresses] : summaryAddressesByEnsemble() )
{
ensemble->prefetchSummaryData( addresses );
}
}
//--------------------------------------------------------------------------------------------------
/// Say that this plot is waiting for data. A source can load without waiting, and then the curves are drawn
/// with what has arrived so far, leaving a plot looking finished when it is not. Called after every load, so
/// the frame appears when the data is asked for and goes away when the last of it has arrived.
//--------------------------------------------------------------------------------------------------
void RimSummaryPlot::updateLoadingOverlayFrame()
{
if ( !plotWidget() ) return;
bool isWaitingForData = false;
for ( const auto& [ensemble, addresses] : summaryAddressesByEnsemble() )
{
if ( ensemble->isSummaryDataPending( addresses ) )
{
isWaitingForData = true;
break;
}
}
if ( !isWaitingForData )
{
if ( m_loadingOverlayFrame )
{
plotWidget()->removeOverlayFrame( m_loadingOverlayFrame );
delete m_loadingOverlayFrame;
m_loadingOverlayFrame = nullptr;
}
return;
}
if ( !m_loadingOverlayFrame )
{
m_loadingOverlayFrame = new RiuDraggableOverlayFrame( plotWidget()->getParentForOverlay(), plotWidget()->overlayMargins() );
m_loadingOverlayFrame->setAnchorCorner( RiuDraggableOverlayFrame::AnchorCorner::TopLeft );
auto* spinnerFrame = new RiuSpinnerOverlayContentFrame( m_loadingOverlayFrame );
m_loadingOverlayFrame->setContentFrame( spinnerFrame );
spinnerFrame->setText( "Loading" );
}
plotWidget()->addOverlayFrame( m_loadingOverlayFrame );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
@@ -1815,6 +1920,8 @@ void RimSummaryPlot::onLoadDataAndUpdate()
auto plotWindow = firstAncestorOrThisOfType<RimMultiPlot>();
if ( plotWindow == nullptr ) updateDockWindowVisibility();
prefetchSummaryData();
if ( m_summaryCurveCollection )
{
m_summaryCurveCollection->loadDataAndUpdate( false );
@@ -1861,6 +1968,8 @@ void RimSummaryPlot::onLoadDataAndUpdate()
updateAxes();
updateStackedCurveData();
updateLoadingOverlayFrame();
}
//--------------------------------------------------------------------------------------------------
ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h
+9 -2
View File
@@ -56,6 +56,7 @@ class RimSummaryTimeAxisProperties;
class RimPlotAxisPropertiesInterface;
class RimPlotAxisProperties;
class RiuSummaryQwtPlot;
class RiuDraggableOverlayFrame;
class RimSummaryNameHelper;
class RimSummaryPlotNameHelper;
class RimPlotTemplateFileItem;
@@ -150,6 +151,8 @@ public:
void deleteAllSummaryCurves();
RimSummaryCurveCollection* summaryCurveCollection() const;
std::map<RimSummaryEnsemble*, std::vector<RifEclipseSummaryAddress>> summaryAddressesByEnsemble() const;
void updatePlotTitle();
const RimSummaryNameHelper* activePlotTitleHelperAllCurves() const;
@@ -250,6 +253,9 @@ protected:
void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override;
void onLoadDataAndUpdate() override;
void prefetchSummaryData();
void updateLoadingOverlayFrame();
bool handleGlobalKeyEvent( QKeyEvent* keyEvent ) override;
private slots:
@@ -341,8 +347,9 @@ private:
caf::PdmChildArrayField<RimPlotAxisPropertiesInterface*> m_axisPropertiesArray;
QPointer<RiuSummaryPlot> m_summaryPlot;
std::unique_ptr<QwtPlotTextLabel> m_plotInfoLabel;
QPointer<RiuSummaryPlot> m_summaryPlot;
QPointer<RiuDraggableOverlayFrame> m_loadingOverlayFrame;
std::unique_ptr<QwtPlotTextLabel> m_plotInfoLabel;
std::unique_ptr<RimSummaryPlotNameHelper> m_nameHelperAllCurves;
caf::PdmChildField<RimSummaryPlotSourceStepping*> m_sourceStepping;
ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp
+303 -42
View File
@@ -30,19 +30,61 @@
#include "RifEclipseSummaryAddress.h"
#include "Cloud/RimCloudDataSourceCollection.h"
#include "RimProject.h"
#include "RimSummaryCaseMainCollection.h"
#include "RimSummaryCaseSumo.h"
#include "RimSummaryCurve.h"
#include "RimSummaryMultiPlot.h"
#include "RimSummaryPlot.h"
#include "RimSumoDataSource.h"
#include <arrow/type_fwd.h>
#include "RiuPlotCurve.h"
#include <arrow/type_fwd.h>
#include <arrow/util/key_value_metadata.h>
#include <algorithm>
#include <map>
#include <memory>
#include <optional>
CAF_PDM_SOURCE_INIT( RimSummaryEnsembleSumo, "RimSummaryEnsembleSumo" );
namespace
{
//--------------------------------------------------------------------------------------------------
/// Whether any curve of the plot holds samples. A plot with none has nothing to fit its axes to, which is the
/// state a plot is left in when it is created for data that has not arrived yet.
//--------------------------------------------------------------------------------------------------
bool hasCurveSamples( const RimSummaryPlot* summaryPlot )
{
for ( const RimSummaryCurve* curve : summaryPlot->summaryAndEnsembleCurves() )
{
if ( curve && curve->plotCurve() && curve->plotCurve()->numSamples() > 0 ) return true;
}
return false;
}
//--------------------------------------------------------------------------------------------------
/// Whether any other plot of the same window already shows something. The plots of a window share their time
/// axis, so a window that shows data has a range that fitting one plot would move for all of them.
//--------------------------------------------------------------------------------------------------
bool windowShowsData( RimSummaryPlot* summaryPlot )
{
auto multiPlot = summaryPlot->firstAncestorOrThisOfType<RimSummaryMultiPlot>();
if ( !multiPlot ) return false;
for ( const RimSummaryPlot* plot : multiPlot->summaryPlots() )
{
if ( plot == summaryPlot ) continue;
if ( hasCurveSamples( plot ) ) return true;
}
return false;
}
//--------------------------------------------------------------------------------------------------
/// Read an integer column of any width as int64. The bit width of a column is decided by the producer
/// of the parquet file, and can not be assumed to be a specific type.
@@ -75,12 +117,36 @@ std::optional<std::vector<double>> readFloatingPointColumn( const std::shared_pt
return {};
}
//--------------------------------------------------------------------------------------------------
/// The unit of a summary vector, as written by the producer of the parquet file into the metadata of the
/// column holding the values. Empty when the file carries no unit for the column.
//--------------------------------------------------------------------------------------------------
std::string unitFromTableColumn( const std::shared_ptr<arrow::Table>& table, const std::string& columnName )
{
if ( !table || !table->schema() ) return {};
auto field = table->schema()->GetFieldByName( columnName );
if ( !field || !field->HasMetadata() ) return {};
auto metadata = field->metadata();
for ( int64_t i = 0; i < metadata->size(); i++ )
{
if ( QString::fromStdString( metadata->key( i ) ).compare( "unit", Qt::CaseInsensitive ) == 0 )
{
return metadata->value( i );
}
}
return {};
}
} // namespace
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
RimSummaryEnsembleSumo::RimSummaryEnsembleSumo()
: summaryDataLoaded( this )
{
CAF_PDM_InitObject( "Sumo Ensemble", ":/SummaryCase.svg", "", "The Base Class for all Summary Cases" );
@@ -93,6 +159,8 @@ RimSummaryEnsembleSumo::RimSummaryEnsembleSumo()
setAsEnsemble( true );
m_sumoConnector = RiaApplication::instance()->makeSumoConnector();
m_lifetimeToken = std::make_shared<bool>( true );
}
//--------------------------------------------------------------------------------------------------
@@ -108,8 +176,19 @@ void RimSummaryEnsembleSumo::setSumoDataSource( RimSumoDataSource* sumoDataSourc
//--------------------------------------------------------------------------------------------------
std::string RimSummaryEnsembleSumo::unitName( const RifEclipseSummaryAddress& resultAddress )
{
// TODO: Not implemented yet. Need to get the unit name from the Sumo data source
return {};
if ( !m_sumoDataSource() ) return {};
// Only the table already fetched for this vector is looked at. Asking for a unit must not put a request on
// its way: it is read while the plot axes are built, and the curves that trigger the load are drawn again
// once the data arrives, which is when the unit becomes available too.
const auto columnName = resultAddress.toEclipseTextAddress();
const auto parquetKey =
ParquetKey{ m_sumoDataSource()->caseId(), m_sumoDataSource()->ensembleName(), QString::fromStdString( columnName ), false };
auto it = m_parquetTable.find( parquetKey );
if ( it == m_parquetTable.end() ) return {};
return unitFromTableColumn( it->second, columnName );
}
//--------------------------------------------------------------------------------------------------
@@ -180,53 +259,229 @@ void RimSummaryEnsembleSumo::updateName( const std::set<QString>& existingEnsemb
//--------------------------------------------------------------------------------------------------
void RimSummaryEnsembleSumo::loadSummaryData( const RifEclipseSummaryAddress& resultAddress )
{
if ( resultAddress.isStatistics() ) return;
loadSummaryData( std::vector<RifEclipseSummaryAddress>{ resultAddress } );
}
// An address without a vector name has no blob to fetch. The special time address used as curve
// x-axis is one such address. Requesting it would produce a URL with an empty vector segment, which
// the service answers with 404, surfacing as a misleading "parquet file size is 0 bytes" error.
if ( resultAddress.vectorName().empty() ) return;
if ( !m_sumoDataSource() ) return;
auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() );
auto sumoCaseId = m_sumoDataSource->caseId();
auto sumoEnsembleName = m_sumoDataSource->ensembleName();
auto key = ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false };
if ( m_parquetTable.find( key ) == m_parquetTable.end() )
{
auto contents = loadParquetData( key );
RiaLogging::debug( std::format( "Load Summary Data. Contents size: {}", contents.size() ) );
std::shared_ptr<arrow::Table> table = readParquetTable( contents, QString::fromStdString( resultAddress.uiText() ) );
m_parquetTable[key] = table;
distributeDataToRealizations( resultAddress, table );
}
auto parametersKey = ParquetKey{ sumoCaseId, sumoEnsembleName, "", true };
if ( m_parquetTable.find( parametersKey ) == m_parquetTable.end() )
{
auto contents = m_sumoConnector->requestParametersParquetDataBlocking( sumoCaseId, sumoEnsembleName );
RiaLogging::debug( std::format( "Load ensemble parameter sensitivities. Contents size: {}", contents.size() ) );
std::shared_ptr<arrow::Table> table = readParquetTable( contents, QString( "%1 parameter sensitivities" ).arg( sumoEnsembleName ) );
m_parquetTable[parametersKey] = table;
distributeParametersDataToRealizations( table );
}
//--------------------------------------------------------------------------------------------------
/// Load several vectors at once. Each vector is one parquet blob covering every realization, so the
/// addresses that are not cached yet are fetched as one concurrent batch rather than one after another.
/// That matters because a vector the service has not aggregated yet is produced on demand by the request
/// asking for it, so fetching serially costs the sum of those aggregations.
//--------------------------------------------------------------------------------------------------
void RimSummaryEnsembleSumo::loadSummaryData( const std::vector<RifEclipseSummaryAddress>& resultAddresses )
{
// Nothing is fetched while the caller waits. A curve asking for values it does not have yet gets none,
// the request is put on its way, and the curve is drawn again once it arrives. Waiting here instead
// stopped the application for as long as the service took, which for a vector it has not aggregated yet
// is a good while.
prefetchSummaryData( resultAddresses );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QByteArray RimSummaryEnsembleSumo::loadParquetData( const ParquetKey& parquetKey )
void RimSummaryEnsembleSumo::loadEnsembleParameters()
{
if ( !m_sumoConnector ) return {};
if ( !m_sumoDataSource() || !m_sumoConnector ) return;
return m_sumoConnector->requestParquetDataBlocking( SumoCaseId( parquetKey.caseId ), parquetKey.ensembleId, parquetKey.vectorName );
auto sumoCaseId = m_sumoDataSource->caseId();
auto sumoEnsembleName = m_sumoDataSource->ensembleName();
auto parametersKey = ParquetKey{ sumoCaseId, sumoEnsembleName, "", true };
if ( m_parquetTable.find( parametersKey ) != m_parquetTable.end() ) return;
if ( m_pendingVectors.find( parametersKey ) != m_pendingVectors.end() ) return;
// Asked for without waiting, like the vectors. The service aggregates the parameters on demand too, so
// the first request for them can take a while, and it used to be made from inside the read of a curve
// value: dropping a vector into a plot stopped the application until the parameters had been fetched.
m_pendingVectors[parametersKey] = RifEclipseSummaryAddress();
std::weak_ptr<bool> isAlive = m_lifetimeToken;
m_sumoConnector->summary().parameterDataAsync( sumoCaseId,
sumoEnsembleName,
[this, isAlive, parametersKey]( const QByteArray& contents )
{
if ( isAlive.expired() ) return;
onParameterDataReceived( parametersKey, contents );
} );
}
//--------------------------------------------------------------------------------------------------
/// The ensemble parameters have arrived. Called on the thread owning the user interface.
//--------------------------------------------------------------------------------------------------
void RimSummaryEnsembleSumo::onParameterDataReceived( const ParquetKey& parquetKey, const QByteArray& contents )
{
auto it = m_pendingVectors.find( parquetKey );
// No longer wanted: the data source changed, or the cache was cleared, while this was on its way.
if ( it == m_pendingVectors.end() ) return;
m_pendingVectors.erase( it );
RiaLogging::debug( std::format( "Load ensemble parameter sensitivities. Contents size: {}", contents.size() ) );
std::shared_ptr<arrow::Table> table = readParquetTable( contents, QString( "%1 parameter sensitivities" ).arg( parquetKey.ensembleId ) );
m_parquetTable[parquetKey] = table;
distributeParametersDataToRealizations( table );
updatePlotsUsingThisEnsemble();
}
//--------------------------------------------------------------------------------------------------
/// Ask for everything the plots are about to read, and return without waiting for any of it. Each vector is
/// taken in as it arrives, so a plot shows the vectors that are ready while the rest are still on their way
/// rather than staying blank until the slowest one is done.
///
/// A vector still on its way is reported as having no data, and the curves using it are drawn empty. They are
/// redrawn when it arrives, see onVectorDataReceived.
//--------------------------------------------------------------------------------------------------
void RimSummaryEnsembleSumo::prefetchSummaryData( const std::vector<RifEclipseSummaryAddress>& resultAddresses )
{
if ( !m_sumoDataSource() || !m_sumoConnector ) return;
auto sumoCaseId = m_sumoDataSource->caseId();
auto sumoEnsembleName = m_sumoDataSource->ensembleName();
std::vector<QString> vectorNamesToFetch;
for ( const auto& resultAddress : resultAddresses )
{
if ( resultAddress.isStatistics() ) continue;
if ( resultAddress.vectorName().empty() ) continue;
auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() );
auto key = ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false };
if ( m_parquetTable.find( key ) != m_parquetTable.end() ) continue;
if ( m_pendingVectors.find( key ) != m_pendingVectors.end() ) continue;
m_pendingVectors[key] = resultAddress;
vectorNamesToFetch.push_back( resultText );
}
// The parameters belong to the ensemble rather than to any one vector, and are wanted as soon as
// anything of it is read. Asked for here so they travel alongside the vectors.
loadEnsembleParameters();
if ( vectorNamesToFetch.empty() ) return;
std::weak_ptr<bool> isAlive = m_lifetimeToken;
m_sumoConnector->summary().vectorDataAsync( sumoCaseId,
sumoEnsembleName,
vectorNamesToFetch,
[this, isAlive, sumoCaseId, sumoEnsembleName]( const QString& vectorName,
const QByteArray& contents )
{
// The request outlived the ensemble that asked for it.
if ( isAlive.expired() ) return;
onVectorDataReceived( ParquetKey{ sumoCaseId, sumoEnsembleName, vectorName, false },
contents );
} );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
bool RimSummaryEnsembleSumo::isSummaryDataPending( const std::vector<RifEclipseSummaryAddress>& resultAddresses ) const
{
if ( m_pendingVectors.empty() || !m_sumoDataSource() ) return false;
auto sumoCaseId = m_sumoDataSource()->caseId();
auto sumoEnsembleName = m_sumoDataSource()->ensembleName();
for ( const auto& resultAddress : resultAddresses )
{
auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() );
if ( m_pendingVectors.contains( ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false } ) ) return true;
}
return false;
}
//--------------------------------------------------------------------------------------------------
/// One requested vector has arrived. Called on the thread owning the user interface, once per vector.
//--------------------------------------------------------------------------------------------------
void RimSummaryEnsembleSumo::onVectorDataReceived( const ParquetKey& parquetKey, const QByteArray& contents )
{
auto it = m_pendingVectors.find( parquetKey );
// No longer wanted: the data source changed, or the cache was cleared, while this was on its way.
if ( it == m_pendingVectors.end() ) return;
const auto resultAddress = it->second;
m_pendingVectors.erase( it );
RiaLogging::debug( std::format( "Load Summary Data. Contents size: {}", contents.size() ) );
// Empty contents mean the request failed. The empty result is stored like any other, so a failure is not
// retried on every redraw, matching what a failed blocking load does.
std::shared_ptr<arrow::Table> table = readParquetTable( contents, QString::fromStdString( resultAddress.uiText() ) );
m_parquetTable[parquetKey] = table;
distributeDataToRealizations( resultAddress, table );
loadEnsembleParameters();
updatePlotsUsingThisEnsemble();
}
//--------------------------------------------------------------------------------------------------
/// Redraw with what has arrived so far. The curves read their values again, those still waiting for data come
/// back empty, and the replot itself is coalesced by the redraw scheduler.
///
/// A plot is created and fitted to its data in one go. Fetching from Sumo does not wait, so a plot created for
/// data still on its way was fitted while it had none, and kept that range. Such a plot is fitted here when its
/// first data arrives, which is what creation would have done had the data been there.
///
/// Only a plot that is still empty is fitted. A plot that already shows something has a range the user may have
/// chosen, and loading another vector into it must not throw that away.
//--------------------------------------------------------------------------------------------------
void RimSummaryEnsembleSumo::updatePlotsUsingThisEnsemble()
{
// Loading a plot can bring in the next vector, which asks for this update again. Finish the pass that is
// running and repeat it afterwards, rather than reloading plots from inside their own load.
if ( m_isUpdatingPlots )
{
m_hasMissedPlotUpdate = true;
return;
}
m_isUpdatingPlots = true;
do
{
m_hasMissedPlotUpdate = false;
for ( RimSummaryPlot* summaryPlot : RimProject::current()->descendantsOfType<RimSummaryPlot>() )
{
if ( !summaryPlot->summaryAddressesByEnsemble().contains( this ) ) continue;
// Fit only a plot that has nothing to show and sits in a window where nothing else does either.
// Plots of a window share their time axis, so fitting one moves them all, and a window already
// showing data has a range the user may have chosen and must keep. A plot added to such a window
// takes the shared range instead, which is what happens when its data comes from the cache.
const bool needsInitialFit = !hasCurveSamples( summaryPlot ) && !windowShowsData( summaryPlot );
summaryPlot->loadDataAndUpdate();
// zoomAll turns the automatic range back on, the one the fit onto an empty plot switched off.
if ( needsInitialFit ) summaryPlot->zoomAll();
summaryPlot->scheduleReplotIfVisible();
}
} while ( m_hasMissedPlotUpdate );
m_isUpdatingPlots = false;
// Sent after the guard is released, so a listener reloading something is not treated as a nested update.
// The plots of the project are already done above; this reaches the views held outside it.
summaryDataLoaded.send();
}
//--------------------------------------------------------------------------------------------------
@@ -354,6 +609,8 @@ void RimSummaryEnsembleSumo::distributeDataToRealizations( const RifEclipseSumma
RiaLogging::warning( "Failed to find values column" );
return;
}
RiaLogging::debug( std::format( "Unit of '{}': '{}'", columnName, unitFromTableColumn( table, columnName ) ) );
}
// find unique realizations
@@ -654,6 +911,10 @@ void RimSummaryEnsembleSumo::clearCachedData()
{
m_resultAddresses.clear();
m_parquetTable.clear();
// Anything still on its way belongs to the data just thrown away. Forgetting it here makes those replies
// drop their contents on arrival, and lets the vectors be requested again if they are still wanted.
m_pendingVectors.clear();
}
//--------------------------------------------------------------------------------------------------
ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h
+23 -2
View File
@@ -57,6 +57,10 @@ class RimSummaryEnsembleSumo : public RimSummaryEnsemble
{
CAF_PDM_HEADER_INIT;
public:
// Sent when data asked for earlier has arrived, for views the reload below cannot reach.
caf::Signal<> summaryDataLoaded;
public:
RimSummaryEnsembleSumo();
@@ -65,12 +69,15 @@ public:
void onRealizationSelectionChanged();
void loadSummaryData( const RifEclipseSummaryAddress& resultAddress );
void loadSummaryData( const std::vector<RifEclipseSummaryAddress>& resultAddresses );
std::string unitName( const RifEclipseSummaryAddress& resultAddress );
RiaDefines::EclipseUnitSystem unitSystem() const;
std::set<RifEclipseSummaryAddress> allResultAddresses() const;
std::pair<std::string, std::string> nameKeys() const override;
void updateName( const std::set<QString>& existingEnsembleNames ) override;
void prefetchSummaryData( const std::vector<RifEclipseSummaryAddress>& resultAddresses ) override;
bool isSummaryDataPending( const std::vector<RifEclipseSummaryAddress>& resultAddresses ) const override;
protected:
void onLoadDataAndUpdate() override;
@@ -83,13 +90,16 @@ private:
void updateResultAddresses();
void clearCachedData();
QByteArray loadParquetData( const ParquetKey& parquetKey );
void distributeDataToRealizations( const RifEclipseSummaryAddress& resultAddress, std::shared_ptr<arrow::Table> table );
void buildMetaData();
void distributeParametersDataToRealizations( std::shared_ptr<arrow::Table> table );
void redistributeCachedDataToRealizations();
void loadEnsembleParameters();
void onVectorDataReceived( const ParquetKey& parquetKey, const QByteArray& contents );
void onParameterDataReceived( const ParquetKey& parquetKey, const QByteArray& contents );
void updatePlotsUsingThisEnsemble();
static std::shared_ptr<arrow::Table> readParquetTable( const QByteArray& contents, const QString& messageTag );
@@ -100,4 +110,15 @@ private:
std::set<RifEclipseSummaryAddress> m_resultAddresses;
std::map<ParquetKey, std::shared_ptr<arrow::Table>> m_parquetTable;
// The vectors requested but not yet arrived, and the address each belongs to. A vector in here is not
// requested again, and is reported as having no data yet rather than being waited for.
std::map<ParquetKey, RifEclipseSummaryAddress> m_pendingVectors;
// Held by the callbacks of requests still on their way. They check it before touching this object, so a
// reply arriving after the ensemble is gone is dropped instead of writing into freed memory.
std::shared_ptr<bool> m_lifetimeToken;
bool m_isUpdatingPlots = false;
bool m_hasMissedPlotUpdate = false;
};
ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp
+143
View File
@@ -21,9 +21,11 @@
#include "RiaFontCache.h"
#include "RiaPreferences.h"
#include <QHBoxLayout>
#include <QLabel>
#include <QPainter>
#include <QTextDocument>
#include <QTimer>
#include <QVBoxLayout>
//--------------------------------------------------------------------------------------------------
@@ -101,3 +103,144 @@ void RiuTextOverlayContentFrame::updateLabelFont()
font.setPointSize( caf::FontTools::pointSizeFromEnum( RiaPreferences::current()->defaultPlotFontSize() ) );
m_textLabel->setFont( font );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
RiuSpinnerOverlayContentFrame::RiuSpinnerOverlayContentFrame( QWidget* parent /*= nullptr */ )
: RiuAbstractOverlayContentFrame( parent )
{
QHBoxLayout* layout = new QHBoxLayout( this );
// Room for the spinner, which is painted rather than laid out: it has no content of its own to size it,
// and reserving the space keeps it from ending up under the text.
layout->setContentsMargins( 4 + spinnerSize() + spinnerMargin(), 4, 4, 4 );
m_textLabel = new QLabel;
layout->addWidget( m_textLabel );
m_animationTimer = new QTimer( this );
m_animationTimer->setInterval( 50 );
QObject::connect( m_animationTimer,
&QTimer::timeout,
this,
[this]()
{
m_angleDegrees = ( m_angleDegrees + 30 ) % 360;
update();
} );
updateLabelFont();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiuSpinnerOverlayContentFrame::setText( const QString& text )
{
m_textLabel->setText( text );
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
int RiuSpinnerOverlayContentFrame::spinnerSize()
{
return 14;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
int RiuSpinnerOverlayContentFrame::spinnerMargin()
{
return 6;
}
//--------------------------------------------------------------------------------------------------
/// Animate only while on screen. A frame taken off a plot keeps its timer, and a timer left running would
/// wake the application up several times a second for something nobody is looking at.
//--------------------------------------------------------------------------------------------------
void RiuSpinnerOverlayContentFrame::showEvent( QShowEvent* event )
{
RiuAbstractOverlayContentFrame::showEvent( event );
m_animationTimer->start();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiuSpinnerOverlayContentFrame::hideEvent( QHideEvent* event )
{
RiuAbstractOverlayContentFrame::hideEvent( event );
m_animationTimer->stop();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiuSpinnerOverlayContentFrame::paintEvent( QPaintEvent* event )
{
RiuAbstractOverlayContentFrame::paintEvent( event );
QPainter painter( this );
drawSpinner( &painter, QPoint( 4, ( height() - spinnerSize() ) / 2 ) );
}
//--------------------------------------------------------------------------------------------------
/// An arc left open at one end, turned a step further on every tick. The gap is what makes the turning
/// visible: a full circle would look the same at every angle.
//--------------------------------------------------------------------------------------------------
void RiuSpinnerOverlayContentFrame::drawSpinner( QPainter* painter, const QPoint& topLeft ) const
{
painter->save();
painter->setRenderHint( QPainter::Antialiasing );
QPen pen( palette().color( QPalette::WindowText ), 2.0, Qt::SolidLine, Qt::RoundCap );
painter->setPen( pen );
painter->setBrush( Qt::NoBrush );
// Qt angles are in sixteenths of a degree and turn counterclockwise, so the sign makes the arc turn the
// way a clock does. Inset by the pen width, otherwise the stroke is drawn half outside the rectangle.
const QRect arcRect( topLeft.x() + 1, topLeft.y() + 1, spinnerSize() - 2, spinnerSize() - 2 );
painter->drawArc( arcRect, -m_angleDegrees * 16, 300 * 16 );
painter->restore();
}
//--------------------------------------------------------------------------------------------------
/// Drawn into snapshots as it looks at this moment. There is no animation in a still image, but leaving the
/// spinner out would make a plot that was still loading look finished.
//--------------------------------------------------------------------------------------------------
void RiuSpinnerOverlayContentFrame::renderTo( QPainter* painter, const QRect& targetRect )
{
updateLabelFont();
painter->save();
painter->translate( targetRect.topLeft() );
drawSpinner( painter, QPoint( 4, ( targetRect.height() - spinnerSize() ) / 2 ) );
painter->translate( contentsMargins().left(), contentsMargins().top() );
painter->setFont( m_textLabel->font() );
QTextDocument td;
td.setDefaultFont( m_textLabel->font() );
td.setHtml( m_textLabel->text() );
td.drawContents( painter );
painter->restore();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiuSpinnerOverlayContentFrame::updateLabelFont()
{
QFont font = m_textLabel->font();
font.setPointSize( caf::FontTools::pointSizeFromEnum( RiaPreferences::current()->defaultPlotFontSize() ) );
m_textLabel->setFont( font );
}
ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h
+32
View File
@@ -22,6 +22,7 @@
#include <QString>
class QLabel;
class QTimer;
class RiuAbstractOverlayContentFrame : public QFrame
{
@@ -51,3 +52,34 @@ private:
private:
QPointer<QLabel> m_textLabel;
};
//==================================================================================================
/// Says that something is going on, for work that finishes on its own and reports no progress along the
/// way. The animation is driven by a timer that only runs while the frame is visible, so a frame that has
/// been taken off a plot costs nothing.
//==================================================================================================
class RiuSpinnerOverlayContentFrame : public RiuAbstractOverlayContentFrame
{
Q_OBJECT
public:
RiuSpinnerOverlayContentFrame( QWidget* parent = nullptr );
void setText( const QString& text );
void renderTo( QPainter* painter, const QRect& targetRect ) override;
protected:
void paintEvent( QPaintEvent* event ) override;
void showEvent( QShowEvent* event ) override;
void hideEvent( QHideEvent* event ) override;
private:
void drawSpinner( QPainter* painter, const QPoint& topLeft ) const;
static int spinnerSize();
static int spinnerMargin();
void updateLabelFont();
private:
QPointer<QLabel> m_textLabel;
QTimer* m_animationTimer = nullptr;
int m_angleDegrees = 0;
};
ApplicationLibCode/UserInterface/RiuMessagePanel.cpp
+31 -3
View File
@@ -26,6 +26,7 @@
#include "cafStyleSheetTools.h"
#include <QCoreApplication>
#include <QMenu>
#include <QPlainTextEdit>
#include <QThread>
@@ -205,6 +206,22 @@ void RiuMessagePanelLogger::debug( const char* message )
writeToMessagePanel( RILogLevel::RI_LL_DEBUG, message );
}
//--------------------------------------------------------------------------------------------------
/// Deliver the messages handed over from other threads. Only the queued addMessage calls above are posted to
/// a panel, so this writes out the pending log messages and nothing else. Must be called from the thread
/// owning the panels.
//--------------------------------------------------------------------------------------------------
void RiuMessagePanelLogger::flushPendingMessages()
{
for ( auto& panel : m_messagePanels )
{
if ( panel && panel->thread() == QThread::currentThread() )
{
QCoreApplication::sendPostedEvents( panel, QEvent::MetaCall );
}
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
@@ -219,13 +236,24 @@ void RiuMessagePanelLogger::writeToMessagePanel( RILogLevel messageLevel, const
{
if ( panel )
{
// Make sure we only output messages for the GUI-thread.
// We can loose some messages, but we avoid updating UI from a different thread that will cause asserts and
// potential crashes
if ( panel->thread() == QThread::currentThread() )
{
panel->addMessage( messageLevel, message );
}
else
{
// The panel can only be touched from the thread owning it. Hand the message over instead of
// dropping it, so messages logged from a worker thread still reach the panel. The text is
// copied into the queued call, as the caller owns the buffer.
const QString messageText = QString::fromUtf8( message );
QMetaObject::invokeMethod(
panel,
[panel, messageLevel, messageText]()
{
if ( panel ) panel->addMessage( messageLevel, messageText );
},
Qt::QueuedConnection );
}
}
}
}
ApplicationLibCode/UserInterface/RiuMessagePanel.h
+2
View File
@@ -70,6 +70,8 @@ public:
void info( const char* message ) override;
void debug( const char* message ) override;
void flushPendingMessages() override;
private:
void writeToMessagePanel( RILogLevel messageLevel, const char* message );