OilfieldToolsNet/ResInsight
4
0
Fork 0
mirror of https://github.com/OPM/ResInsight.git synced 2025-02-25 18:55:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

#2583 System : Move regression test framework to separate class

Browse Source
This commit is contained in:
Magne Sjaastad 2018-03-08 10:10:15 +01:00
parent 09246b722a
commit 4f7291c95d
10 changed files with 667 additions and 503 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

500
ApplicationCode/Application/RiaApplication.cpp
View File

@ -22,99 +22,65 @@
#include "RiaArgumentParser.h"
#include "RiaBaseDefs.h"
#include "RiaImageCompareReporter.h"
#include "RiaImageFileCompare.h"
#include "RiaImportEclipseCaseTools.h"
#include "RiaLogging.h"
#include "RiaPreferences.h"
#include "RiaProjectModifier.h"
#include "RiaRegressionTest.h"
#include "RiaSocketServer.h"
#include "RiaVersionInfo.h"
#include "RiaViewRedrawScheduler.h"
#include "RigGridManager.h"
#include "RigEclipseCaseData.h"
#include "RicImportInputEclipseCaseFeature.h"
#include "RicImportSummaryCaseFeature.h"
#include "ExportCommands/RicSnapshotAllViewsToFileFeature.h"
#include "Rim2dIntersectionViewCollection.h"
#include "Rim3dOverlayInfoConfig.h"
#include "RimCaseCollection.h"
#include "RimCellEdgeColors.h"
#include "RimCellRangeFilterCollection.h"
#include "RimCommandObject.h"
#include "RimEclipseCase.h"
#include "RimEclipseCaseCollection.h"
#include "RimEclipseView.h"
#include "RimFaultInViewCollection.h"
#include "RimFlowCharacteristicsPlot.h"
#include "RimFlowPlotCollection.h"
#include "RimFormationNamesCollection.h"
#include "RimRftPlotCollection.h"
#include "RimPltPlotCollection.h"
#include "RimEclipseCase.h"
#include "RimFractureTemplateCollection.h"
#include "RimGeoMechCase.h"
#include "RimGeoMechCellColors.h"
#include "RimGeoMechModels.h"
#include "RimGeoMechView.h"
#include "RimGeoMechView.h"
#include "RimIdenticalGridCaseGroup.h"
#include "RimMainPlotCollection.h"
#include "RimObservedData.h"
#include "RimObservedDataCollection.h"
#include "RimOilField.h"
#include "RimPltPlotCollection.h"
#include "RimProject.h"
#include "RimReservoirCellResultsStorage.h"
#include "RimScriptCollection.h"
#include "RimRftPlotCollection.h"
#include "RimStimPlanColors.h"
#include "RimSummaryCase.h"
#include "RimSummaryCaseMainCollection.h"
#include "RimSummaryCrossPlotCollection.h"
#include "RimSummaryCurve.h"
#include "RimSummaryPlot.h"
#include "RimSummaryPlotCollection.h"
#include "RimViewLinker.h"
#include "RimViewLinkerCollection.h"
#include "RimWellAllocationPlot.h"
#include "RimWellLogFile.h"
#include "RimWellLogPlot.h"
#include "RimWellLogPlotCollection.h"
#include "RimWellPath.h"
#include "RimWellPathCollection.h"
#include "RimWellRftPlot.h"
#include "RimWellPathFracture.h"
#include "RimWellPltPlot.h"
#include "RimWellRftPlot.h"
#include "RiuMainPlotWindow.h"
#include "RiuMainWindow.h"
#include "RiuProcessMonitor.h"
#include "RiuRecentFileActionProvider.h"
#include "RiuSelectionManager.h"
#include "RiuSummaryQwtPlot.h"
#include "RiuViewer.h"
#include "RiuWellLogPlot.h"
#include "RiuWellAllocationPlot.h"
#include "RiuFlowCharacteristicsPlot.h"
#include "RimFractureTemplateCollection.h"
#include "RimWellPathFracture.h"
#include "RimStimPlanColors.h"
#include "RicImportInputEclipseCaseFeature.h"
#include "RicImportSummaryCaseFeature.h"
#include "ExportCommands/RicSnapshotViewToFileFeature.h"
#include "ExportCommands/RicSnapshotAllPlotsToFileFeature.h"
#include "ExportCommands/RicSnapshotAllViewsToFileFeature.h"
#include "SummaryPlotCommands/RicNewSummaryPlotFeature.h"
#include "RicfCommandFileExecutor.h"
#include "cafFixedAtlasFont.h"
#include "cafAppEnum.h"
#include "cafCeetronPlusNavigation.h"
#include "cafEffectCache.h"
#include "cafPdmFieldCvfColor.h"
#include "cafPdmFieldCvfMat4d.h"
#include "cafEffectGenerator.h"
#include "cafFixedAtlasFont.h"
#include "cafPdmSettings.h"
#include "cafPdmUiModelChangeDetector.h"
#include "cafPdmUiTreeView.h"
@ -126,22 +92,15 @@
#include "cvfProgramOptions.h"
#include "cvfqtUtils.h"
#include <QAction>
#include <QDesktopServices>
#include <QDir>
#include <QFileDialog>
#include <QMdiSubWindow>
#include <QMessageBox>
#include <QTimer>
#include <QUrl>
#include <QTreeView>
#ifdef USE_UNIT_TESTS
#include "gtest/gtest.h"
#ifdef WIN32
#include <fcntl.h>
#endif
#endif // USE_UNIT_TESTS
namespace caf
{
@ -158,14 +117,16 @@ void AppEnum< RiaApplication::RINavigationPolicy >::setUp()
namespace RegTestNames
{
const QString generatedFolderName = "RegTestGeneratedImages";
const QString diffFolderName = "RegTestDiffImages";
const QString baseFolderName = "RegTestBaseImages";
const QString testProjectName = "RegressionTest";
const QString testFolderFilter = "TestCase*";
const QString imageCompareExeName = "compare";
const QString reportFileName = "ResInsightRegressionTestReport.html";
const QString commandFileFilter = "commandfile-*";
const QString generatedFilesFolderName = "RegTestGeneratedFiles";
const QString baseFilesFolderName = "RegTestBaseFiles";
const QString generatedFolderName = "RegTestGeneratedImages";
const QString diffFolderName = "RegTestDiffImages";
const QString baseFolderName = "RegTestBaseImages";
const QString testProjectName = "RegressionTest";
const QString testFolderFilter = "TestCase*";
const QString imageCompareExeName = "compare";
const QString reportFileName = "ResInsightRegressionTestReport.html";
const QString commandFileFilter = "commandfile-*";
};
@ -230,8 +191,6 @@ RiaApplication::RiaApplication(int& argc, char** argv)
// instead of using the application font
m_standardFont = new caf::FixedAtlasFont(caf::FixedAtlasFont::POINT_SIZE_8);
m_runningRegressionTests = false;
m_runningWorkerProcess = false;
m_mainPlotWindow = nullptr;
@ -1538,6 +1497,19 @@ std::vector<QString> RiaApplication::readFileListFromTextFile(QString listFileNa
return fileList;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaApplication::waitUntilCommandObjectsHasBeenProcessed()
{
// Wait until all command objects have completed
while (!m_commandQueueLock.tryLock())
{
processEvents();
}
m_commandQueueLock.unlock();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
@ -1963,317 +1935,6 @@ void RiaApplication::runMultiCaseSnapshots(const QString& templateProjectFileNam
mainWnd->loadWinGeoAndDockToolBarLayout();
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void removeDirectoryWithContent(QDir dirToDelete )
{
QStringList files = dirToDelete.entryList();
for (int fIdx = 0; fIdx < files.size(); ++fIdx)
{
dirToDelete.remove(files[fIdx]);
}
dirToDelete.rmdir(".");
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void logInfoTextWithTimeInSeconds(const QTime& time, const QString& msg)
{
double timeRunning = time.elapsed() / 1000.0;
QString timeText = QString("(%1 s) ").arg(timeRunning, 0, 'f', 1);
RiaLogging::info(timeText + msg);
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaApplication::runRegressionTest(const QString& testRootPath, QStringList* testFilter)
{
m_runningRegressionTests = true;
RiaRegressionTest regressionTestConfig;
regressionTestConfig.readSettingsFromApplicationStore();
QString currentApplicationPath = QDir::currentPath();
if (!regressionTestConfig.folderContainingCompareTool().isEmpty())
{
// Windows Only : The image compare tool requires current working directory to be at the folder
// containing the image compare tool
QDir::setCurrent(regressionTestConfig.folderContainingCompareTool());
}
QString generatedFolderName = RegTestNames::generatedFolderName;
QString diffFolderName = RegTestNames::diffFolderName;
QString baseFolderName = RegTestNames::baseFolderName;
QString regTestProjectName = RegTestNames::testProjectName;
QString regTestFolderFilter = RegTestNames::testFolderFilter;
// Find all sub folders
QDir testDir(testRootPath); // If string is empty it will end up as cwd
testDir.setFilter(QDir::Dirs);
QStringList dirNameFilter;
dirNameFilter.append(regTestFolderFilter);
testDir.setNameFilters(dirNameFilter);
QFileInfoList folderList = testDir.entryInfoList();
if (testFilter && testFilter->size() > 0)
{
QFileInfoList subset;
for (auto fi : folderList)
{
QString path = fi.path();
QString baseName = fi.baseName();
for (auto s : *testFilter)
{
QString trimmed = s.trimmed();
if (baseName.contains(trimmed))
{
subset.push_back(fi);
}
}
}
folderList = subset;
}
// delete diff and generated images
for (int i = 0; i < folderList.size(); ++i)
{
QDir testCaseFolder(folderList[i].filePath());
QDir genDir(testCaseFolder.filePath(generatedFolderName));
removeDirectoryWithContent(genDir);
QDir diffDir(testCaseFolder.filePath(diffFolderName));
removeDirectoryWithContent(diffDir);
QDir baseDir(testCaseFolder.filePath(baseFolderName));
}
// Generate html report
RiaImageCompareReporter imageCompareReporter;
// Minor workaround
// Use registry to define if interactive diff images should be created
// Defined by user in RiaRegressionTest
{
QSettings settings;
bool useInteractiveDiff = settings.value("showInteractiveDiffImages").toBool();
if (useInteractiveDiff)
{
imageCompareReporter.showInteractiveOnly();
}
}
QTime timeStamp;
timeStamp.start();
logInfoTextWithTimeInSeconds(timeStamp, "Starting regression tests\n");
for (int dirIdx = 0; dirIdx < folderList.size(); ++dirIdx)
{
QDir testCaseFolder(folderList[dirIdx].filePath());
QString testFolderName = testCaseFolder.dirName();
QString reportBaseFolderName = testCaseFolder.filePath(baseFolderName);
QString reportGeneratedFolderName = testCaseFolder.filePath(generatedFolderName);
QString reportDiffFolderName = testCaseFolder.filePath(diffFolderName);
imageCompareReporter.addImageDirectoryComparisonSet(testFolderName.toStdString(), reportBaseFolderName.toStdString(), reportGeneratedFolderName.toStdString(), reportDiffFolderName.toStdString());
}
QString htmlReportFileName = testDir.filePath(RegTestNames::reportFileName);
imageCompareReporter.generateHTMLReport(htmlReportFileName.toStdString());
// Open HTML report
QDesktopServices::openUrl(htmlReportFileName);
for (int dirIdx = 0; dirIdx < folderList.size(); ++dirIdx)
{
QDir testCaseFolder(folderList[dirIdx].filePath());
// Detect any command files
QStringList filterList;
filterList << RegTestNames::commandFileFilter;
QFileInfoList commandFileEntries = testCaseFolder.entryInfoList(filterList);
if (!commandFileEntries.empty())
{
QString currentApplicationPath = QDir::current().absolutePath();
// Set current path to the folder containing the command file, as this is required when using file references
// in the command file
QDir::setCurrent(folderList[dirIdx].filePath());
for (const auto& fileInfo : commandFileEntries)
{
QString commandFile = fileInfo.absoluteFilePath();
QFile file(commandFile);
if (!file.open(QIODevice::ReadOnly | QIODevice::Text))
{
RiaLogging::error("Failed to open command file : " + commandFile);
}
else
{
QTextStream in(&file);
RicfCommandFileExecutor::instance()->executeCommands(in);
}
}
QDir::setCurrent(currentApplicationPath);
}
QString projectFileName;
if (testCaseFolder.exists(regTestProjectName + ".rip"))
{
projectFileName = regTestProjectName + ".rip";
}
if (testCaseFolder.exists(regTestProjectName + ".rsp"))
{
projectFileName = regTestProjectName + ".rsp";
}
if (!projectFileName.isEmpty())
{
logInfoTextWithTimeInSeconds(timeStamp, "Initializing test :" + testCaseFolder.absolutePath());
loadProject(testCaseFolder.filePath(projectFileName));
// Wait until all command objects have completed
while (!m_commandQueueLock.tryLock())
{
processEvents();
}
m_commandQueueLock.unlock();
regressionTestConfigureProject();
resizeMaximizedPlotWindows();
QString fullPathGeneratedFolder = testCaseFolder.absoluteFilePath(generatedFolderName);
RicSnapshotAllViewsToFileFeature::exportSnapshotOfAllViewsIntoFolder(fullPathGeneratedFolder);
RicSnapshotAllPlotsToFileFeature::exportSnapshotOfAllPlotsIntoFolder(fullPathGeneratedFolder);
QDir baseDir(testCaseFolder.filePath(baseFolderName));
QDir genDir(testCaseFolder.filePath(generatedFolderName));
QDir diffDir(testCaseFolder.filePath(diffFolderName));
if (!diffDir.exists()) testCaseFolder.mkdir(diffFolderName);
baseDir.setFilter(QDir::Files);
QStringList baseImageFileNames = baseDir.entryList();
for (int fIdx = 0; fIdx < baseImageFileNames.size(); ++fIdx)
{
QString fileName = baseImageFileNames[fIdx];
RiaImageFileCompare imgComparator(RegTestNames::imageCompareExeName);
bool ok = imgComparator.runComparison(genDir.filePath(fileName), baseDir.filePath(fileName), diffDir.filePath(fileName));
if (!ok)
{
qDebug() << "Error comparing :" << imgComparator.errorMessage() << "\n" << imgComparator.errorDetails();
}
}
closeProject();
logInfoTextWithTimeInSeconds(timeStamp, "Completed test :" + testCaseFolder.absolutePath());
}
else
{
RiaLogging::error("Could not find a regression test file named : " + testCaseFolder.absolutePath() + "/" + regTestProjectName + ".rsp");
}
}
RiaLogging::info("\n");
logInfoTextWithTimeInSeconds(timeStamp, "Completed regression tests");
QDir::setCurrent(currentApplicationPath);
m_runningRegressionTests = false;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaApplication::resizeMaximizedPlotWindows()
{
std::vector<RimViewWindow*> viewWindows;
m_project->mainPlotCollection()->descendantsIncludingThisOfType(viewWindows);
for (auto viewWindow : viewWindows)
{
if (viewWindow->isMdiWindow())
{
RimMdiWindowGeometry wndGeo = viewWindow->mdiWindowGeometry();
if (wndGeo.isMaximized)
{
QWidget* viewWidget = viewWindow->viewWidget();
if (viewWidget)
{
QMdiSubWindow* mdiWindow = m_mainPlotWindow->findMdiSubWindow(viewWidget);
if (mdiWindow)
{
mdiWindow->showNormal();
viewWidget->resize(RiaApplication::regressionDefaultImageSize());
}
}
}
}
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaApplication::updateRegressionTest(const QString& testRootPath)
{
// Find all sub folders
QDir testDir(testRootPath); // If string is empty it will end up as cwd
testDir.setFilter(QDir::Dirs);
QStringList dirNameFilter;
dirNameFilter.append(RegTestNames::testFolderFilter);
testDir.setNameFilters(dirNameFilter);
QFileInfoList folderList = testDir.entryInfoList();
for (int i = 0; i < folderList.size(); ++i)
{
QDir testCaseFolder(folderList[i].filePath());
QDir baseDir(testCaseFolder.filePath(RegTestNames::baseFolderName));
removeDirectoryWithContent(baseDir);
testCaseFolder.mkdir(RegTestNames::baseFolderName);
QDir genDir(testCaseFolder.filePath(RegTestNames::generatedFolderName));
QStringList imageFileNames = genDir.entryList();
for (int fIdx = 0; fIdx < imageFileNames.size(); ++fIdx)
{
QString fileName = imageFileNames[fIdx];
QFile::copy(genDir.filePath(fileName), baseDir.filePath(fileName));
}
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
@ -2439,93 +2100,6 @@ void RiaApplication::executeCommandObjects()
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
bool RiaApplication::isRunningRegressionTests() const
{
return m_runningRegressionTests;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaApplication::executeRegressionTests(const QString& regressionTestPath, QStringList* testFilter)
{
RiuMainWindow* mainWnd = RiuMainWindow::instance();
if (mainWnd)
{
mainWnd->hideAllDockWindows();
mainWnd->setDefaultWindowSize();
runRegressionTest(regressionTestPath, testFilter);
mainWnd->loadWinGeoAndDockToolBarLayout();
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaApplication::executeRegressionTests()
{
RiaRegressionTest testConfig;
testConfig.readSettingsFromApplicationStore();
QString testPath = testConfig.regressionTestFolder();
QStringList testFilter = testConfig.testFilter().split(";", QString::SkipEmptyParts);
executeRegressionTests(testPath, &testFilter);
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaApplication::regressionTestConfigureProject()
{
RiuMainWindow* mainWnd = RiuMainWindow::instance();
if (!mainWnd) return;
if (m_project.isNull()) return;
std::vector<RimCase*> projectCases;
m_project->allCases(projectCases);
for (size_t i = 0; i < projectCases.size(); i++)
{
RimCase* cas = projectCases[i];
if (!cas) continue;
std::vector<Rim3dView*> views = cas->views();
for (size_t j = 0; j < views.size(); j++)
{
Rim3dView* riv = views[j];
if (riv && riv->viewer())
{
// Make sure all views are maximized for snapshotting
QMdiSubWindow* subWnd = mainWnd->findMdiSubWindow(riv->viewer()->layoutWidget());
if (subWnd)
{
subWnd->showMaximized();
}
// This size is set to match the regression test reference images
riv->viewer()->setFixedSize(RiaApplication::regressionDefaultImageSize());
}
}
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QSize RiaApplication::regressionDefaultImageSize()
{
return QSize(1000, 745);
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------

42
ApplicationCode/Application/RiaApplication.h
View File

@ -93,22 +93,20 @@ public:
public:
RiaApplication(int& argc, char** argv);
~RiaApplication();
static RiaApplication* instance();
static RiaApplication* instance();
int parseArgumentsAndRunUnitTestsIfRequested();
bool parseArguments();
int parseArgumentsAndRunUnitTestsIfRequested();
bool parseArguments();
void executeRegressionTests(const QString& regressionTestPath, QStringList* testFilter = nullptr);
void executeRegressionTests();
void setActiveReservoirView(Rim3dView*);
Rim3dView* activeReservoirView();
const Rim3dView* activeReservoirView() const;
RimGridView* activeGridView();
void setActiveReservoirView(Rim3dView*);
Rim3dView* activeReservoirView();
const Rim3dView* activeReservoirView() const;
RimGridView* activeGridView();
RimViewWindow* activePlotWindow() const;
RimViewWindow* activePlotWindow() const;
RimProject* project();
RimProject* project();
void createMockModel();
void createResultsMockModel();
@ -141,7 +139,6 @@ public:
void addWellLogsToModel(const QList<QString>& wellLogFilePaths);
void runMultiCaseSnapshots(const QString& templateProjectFileName, std::vector<QString> gridFileNames, const QString& snapshotFolderName);
void runRegressionTest(const QString& testRootPath, QStringList* testFilter = nullptr);
void processNonGuiEvents();
@ -180,8 +177,6 @@ public:
void addCommandObject(RimCommandObject* commandObject);
void executeCommandObjects();
bool isRunningRegressionTests() const;
int launchUnitTests();
int launchUnitTestsWithConsole();
@ -204,9 +199,10 @@ public:
static std::vector<QString> readFileListFromTextFile(QString listFileName);
void waitUntilCommandObjectsHasBeenProcessed();
private:
void onProjectOpenedOrClosed();
void setWindowCaptionFromAppState();
@ -217,25 +213,16 @@ private:
void storeTreeViewState();
void resizeMaximizedPlotWindows();
void updateRegressionTest(const QString& testRootPath);
void regressionTestConfigureProject();
static QSize regressionDefaultImageSize();
friend RiaArgumentParser;
void setHelpText(const QString& helpText);
private slots:
void slotWorkerProcessFinished(int exitCode, QProcess::ExitStatus exitStatus);
void slotWorkerProcessFinished(int exitCode, QProcess::ExitStatus exitStatus);
private:
caf::PdmPointer<Rim3dView> m_activeReservoirView;
caf::PdmPointer<Rim3dView> m_activeReservoirView;
caf::PdmPointer<RimProject> m_project;
RiaSocketServer* m_socketServer;
caf::UiProcess* m_workerProcess;
@ -259,7 +246,6 @@ private:
QMutex m_commandQueueLock;
QString m_helpText;
bool m_runningRegressionTests;
bool m_runningWorkerProcess;

2
ApplicationCode/Application/Tools/CMakeLists_files.cmake
View File

@ -21,6 +21,7 @@ ${CMAKE_CURRENT_LIST_DIR}/RiaProjectFileVersionTools.h
${CMAKE_CURRENT_LIST_DIR}/RiaStringEncodingTools.h
${CMAKE_CURRENT_LIST_DIR}/RiaTextStringTools.h
${CMAKE_CURRENT_LIST_DIR}/RiaTextFileCompare.h
${CMAKE_CURRENT_LIST_DIR}/RiaRegressionTestRunner.h
)
set (SOURCE_GROUP_SOURCE_FILES
@ -45,6 +46,7 @@ ${CMAKE_CURRENT_LIST_DIR}/RiaProjectFileVersionTools.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaStringEncodingTools.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaTextStringTools.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaTextFileCompare.cpp
${CMAKE_CURRENT_LIST_DIR}/RiaRegressionTestRunner.cpp
)
list(APPEND CODE_HEADER_FILES

11
ApplicationCode/Application/Tools/RiaArgumentParser.cpp
View File

@ -18,11 +18,12 @@
#include "RiaArgumentParser.h"
#include "RiaProjectModifier.h"
#include "RiaApplication.h"
#include "RiaPreferences.h"
#include "RiaImportEclipseCaseTools.h"
#include "RiaBaseDefs.h"
#include "RiaImportEclipseCaseTools.h"
#include "RiaPreferences.h"
#include "RiaProjectModifier.h"
#include "RiaRegressionTestRunner.h"
#include "RimProject.h"
@ -115,7 +116,7 @@ bool RiaArgumentParser::parseArguments()
{
CVF_ASSERT(o.valueCount() == 1);
QString regressionTestPath = cvfqt::Utils::toQString(o.value(0));
RiaApplication::instance()->executeRegressionTests(regressionTestPath);
RiaRegressionTestRunner::instance()->executeRegressionTests(regressionTestPath, QStringList());
return false;
}
@ -123,7 +124,7 @@ bool RiaArgumentParser::parseArguments()
{
CVF_ASSERT(o.valueCount() == 1);
QString regressionTestPath = cvfqt::Utils::toQString(o.value(0));
RiaApplication::instance()->updateRegressionTest(regressionTestPath);
RiaRegressionTestRunner::instance()->updateRegressionTest(regressionTestPath);
return false;
}

546
ApplicationCode/Application/Tools/RiaRegressionTestRunner.cpp Normal file
View File

@ -0,0 +1,546 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2018 Statoil ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#include "RiaRegressionTestRunner.h"
#include "RiaApplication.h"
#include "RiaImageCompareReporter.h"
#include "RiaImageFileCompare.h"
#include "RiaLogging.h"
#include "RiaRegressionTest.h"
#include "RiaTextFileCompare.h"
#include "RicfCommandFileExecutor.h"
#include "Rim3dView.h"
#include "RimCase.h"
#include "RimMainPlotCollection.h"
#include "RimProject.h"
#include "RiuMainPlotWindow.h"
#include "RiuMainWindow.h"
#include "RiuViewer.h"
#include "ExportCommands/RicSnapshotAllPlotsToFileFeature.h"
#include "ExportCommands/RicSnapshotAllViewsToFileFeature.h"
#include <QDateTime>
#include <QDebug>
#include <QDesktopServices>
#include <QDir>
#include <QMdiSubWindow>
#include <QSettings>
#include <QString>
#include <QUrl>
namespace RegTestNames
{
const QString generatedFilesFolderName = "RegTestGeneratedFiles";
const QString baseFilesFolderName = "RegTestBaseFiles";
const QString generatedFolderName = "RegTestGeneratedImages";
const QString diffFolderName = "RegTestDiffImages";
const QString baseFolderName = "RegTestBaseImages";
const QString testProjectName = "RegressionTest";
const QString testFolderFilter = "TestCase*";
const QString imageCompareExeName = "compare";
const QString reportFileName = "ResInsightRegressionTestReport.html";
const QString commandFileFilter = "commandfile-*";
}; // namespace RegTestNames
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void logInfoTextWithTimeInSeconds(const QTime& time, const QString& msg)
{
double timeRunning = time.elapsed() / 1000.0;
QString timeText = QString("(%1 s) ").arg(timeRunning, 0, 'f', 1);
RiaLogging::info(timeText + msg);
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
RiaRegressionTestRunner::RiaRegressionTestRunner()
: m_runningRegressionTests(false)
{
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
RiaRegressionTestRunner* RiaRegressionTestRunner::instance()
{
static RiaRegressionTestRunner* singleton = new RiaRegressionTestRunner;
return singleton;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaRegressionTestRunner::runRegressionTest(const QString& testRootPath, const QStringList& testFilter)
{
m_runningRegressionTests = true;
RiaRegressionTest regressionTestConfig;
regressionTestConfig.readSettingsFromApplicationStore();
QString currentApplicationPath = QDir::currentPath();
if (!regressionTestConfig.folderContainingCompareTool().isEmpty())
{
// Windows Only : The image compare tool requires current working directory to be at the folder
// containing the image compare tool
QDir::setCurrent(regressionTestConfig.folderContainingCompareTool());
}
QString generatedFolderName = RegTestNames::generatedFolderName;
QString diffFolderName = RegTestNames::diffFolderName;
QString baseFolderName = RegTestNames::baseFolderName;
QString regTestProjectName = RegTestNames::testProjectName;
QString regTestFolderFilter = RegTestNames::testFolderFilter;
QDir testDir(testRootPath); // If string is empty it will end up as cwd
testDir.setFilter(QDir::Dirs);
QStringList dirNameFilter;
dirNameFilter.append(regTestFolderFilter);
testDir.setNameFilters(dirNameFilter);
QFileInfoList folderList = testDir.entryInfoList();
if (!testFilter.isEmpty())
{
QFileInfoList subset;
for (auto fi : folderList)
{
QString path = fi.path();
QString baseName = fi.baseName();
for (auto s : testFilter)
{
QString trimmed = s.trimmed();
if (baseName.contains(trimmed))
{
subset.push_back(fi);
}
}
}
folderList = subset;
}
// delete diff and generated images
for (int i = 0; i < folderList.size(); ++i)
{
QDir testCaseFolder(folderList[i].filePath());
QDir genDir(testCaseFolder.filePath(generatedFolderName));
removeDirectoryWithContent(genDir);
QDir diffDir(testCaseFolder.filePath(diffFolderName));
removeDirectoryWithContent(diffDir);
QDir baseDir(testCaseFolder.filePath(baseFolderName));
}
// Generate html report
RiaImageCompareReporter imageCompareReporter;
// Minor workaround
// Use registry to define if interactive diff images should be created
// Defined by user in RiaRegressionTest
{
QSettings settings;
bool useInteractiveDiff = settings.value("showInteractiveDiffImages").toBool();
if (useInteractiveDiff)
{
imageCompareReporter.showInteractiveOnly();
}
}
QTime timeStamp;
timeStamp.start();
logInfoTextWithTimeInSeconds(timeStamp, "Starting regression tests\n");
for (int dirIdx = 0; dirIdx < folderList.size(); ++dirIdx)
{
QDir testCaseFolder(folderList[dirIdx].filePath());
QString testFolderName = testCaseFolder.dirName();
QString reportBaseFolderName = testCaseFolder.filePath(baseFolderName);
QString reportGeneratedFolderName = testCaseFolder.filePath(generatedFolderName);
QString reportDiffFolderName = testCaseFolder.filePath(diffFolderName);
imageCompareReporter.addImageDirectoryComparisonSet(testFolderName.toStdString(),
reportBaseFolderName.toStdString(),
reportGeneratedFolderName.toStdString(),
reportDiffFolderName.toStdString());
}
QString htmlReportFileName = testDir.filePath(RegTestNames::reportFileName);
imageCompareReporter.generateHTMLReport(htmlReportFileName.toStdString());
// Open HTML report
QDesktopServices::openUrl(htmlReportFileName);
for (int dirIdx = 0; dirIdx < folderList.size(); ++dirIdx)
{
QDir testCaseFolder(folderList[dirIdx].filePath());
// Detect any command files
QStringList filterList;
filterList << RegTestNames::commandFileFilter;
QFileInfoList commandFileEntries = testCaseFolder.entryInfoList(filterList);
if (!commandFileEntries.empty())
{
QString currentApplicationPath = QDir::current().absolutePath();
// Set current path to the folder containing the command file, as this is required when using file references
// in the command file
QDir::setCurrent(folderList[dirIdx].filePath());
for (const auto& fileInfo : commandFileEntries)
{
QString commandFile = fileInfo.absoluteFilePath();
QFile file(commandFile);
if (!file.open(QIODevice::ReadOnly | QIODevice::Text))
{
RiaLogging::error("Failed to open command file : " + commandFile);
}
else
{
QTextStream in(&file);
RicfCommandFileExecutor::instance()->executeCommands(in);
}
}
QDir::setCurrent(currentApplicationPath);
// Create diff based on generated folders
{
QString html;
RiaRegressionTest regressionTestConfig;
regressionTestConfig.readSettingsFromApplicationStore();
RiaTextFileCompare textFileCompare(regressionTestConfig.folderContainingDiffTool());
QString baseFilesFolderName = testCaseFolder.filePath(RegTestNames::baseFilesFolderName);
QString generatedFilesFolderName = testCaseFolder.filePath(RegTestNames::generatedFilesFolderName);
QFileInfo fib(baseFilesFolderName);
QFileInfo fig(generatedFilesFolderName);
if (fib.exists() && fig.exists())
{
{
QString headerText = testCaseFolder.dirName();
html += "<table>\n";
html += " <tr>\n";
html += " <td colspan=\"3\" bgcolor=\"darkblue\" height=\"40\"> <b><font color=\"white\" size=\"3\"> " + headerText + " </font></b> </td>\n";
html += " </tr>\n";
textFileCompare.runComparison(baseFilesFolderName, generatedFilesFolderName);
QString diff = textFileCompare.diffOutput();
if (diff.isEmpty())
{
html += " <tr>\n";
html += " <td colspan=\"3\" bgcolor=\"lightgray\"> <font color=\"green\">No text diff detected</font> </td> \n";
html += " </tr>\n";
}
else
{
html += " <tr>\n";
html += " <td colspan=\"3\" bgcolor=\"lightgray\"> <font color=\"red\">Text diff detected - output from diff tool : </font> </td> \n";
html += " </tr>\n";
}
// Table end
html += "</table>\n";
if (!diff.isEmpty())
{
html += QString("<code> %1 </code>").arg(diff);
}
}
QFile file(htmlReportFileName);
if (file.open(QIODevice::Append | QIODevice::Text))
{
QTextStream stream(&file);
stream << html;
}
}
}
}
QString projectFileName;
if (testCaseFolder.exists(regTestProjectName + ".rip"))
{
projectFileName = regTestProjectName + ".rip";
}
if (testCaseFolder.exists(regTestProjectName + ".rsp"))
{
projectFileName = regTestProjectName + ".rsp";
}
if (!projectFileName.isEmpty())
{
logInfoTextWithTimeInSeconds(timeStamp, "Initializing test :" + testCaseFolder.absolutePath());
RiaApplication* app = RiaApplication::instance();
app->loadProject(testCaseFolder.filePath(projectFileName));
// Wait until all command objects have completed
app->waitUntilCommandObjectsHasBeenProcessed();
regressionTestConfigureProject();
resizeMaximizedPlotWindows();
QString fullPathGeneratedFolder = testCaseFolder.absoluteFilePath(generatedFolderName);
RicSnapshotAllViewsToFileFeature::exportSnapshotOfAllViewsIntoFolder(fullPathGeneratedFolder);
RicSnapshotAllPlotsToFileFeature::exportSnapshotOfAllPlotsIntoFolder(fullPathGeneratedFolder);
QDir baseDir(testCaseFolder.filePath(baseFolderName));
QDir genDir(testCaseFolder.filePath(generatedFolderName));
QDir diffDir(testCaseFolder.filePath(diffFolderName));
if (!diffDir.exists()) testCaseFolder.mkdir(diffFolderName);
baseDir.setFilter(QDir::Files);
QStringList baseImageFileNames = baseDir.entryList();
for (int fIdx = 0; fIdx < baseImageFileNames.size(); ++fIdx)
{
QString fileName = baseImageFileNames[fIdx];
RiaImageFileCompare imgComparator(RegTestNames::imageCompareExeName);
bool ok = imgComparator.runComparison(
genDir.filePath(fileName), baseDir.filePath(fileName), diffDir.filePath(fileName));
if (!ok)
{
qDebug() << "Error comparing :" << imgComparator.errorMessage() << "\n" << imgComparator.errorDetails();
}
}
app->closeProject();
logInfoTextWithTimeInSeconds(timeStamp, "Completed test :" + testCaseFolder.absolutePath());
}
else
{
RiaLogging::error("Could not find a regression test file named : " + testCaseFolder.absolutePath() + "/" +
regTestProjectName + ".rsp");
}
}
RiaLogging::info("\n");
logInfoTextWithTimeInSeconds(timeStamp, "Completed regression tests");
QDir::setCurrent(currentApplicationPath);
m_runningRegressionTests = false;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaRegressionTestRunner::removeDirectoryWithContent(QDir& dirToDelete)
{
QStringList files = dirToDelete.entryList();
for (int fIdx = 0; fIdx < files.size(); ++fIdx)
{
dirToDelete.remove(files[fIdx]);
}
dirToDelete.rmdir(".");
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaRegressionTestRunner::regressionTestConfigureProject()
{
RiuMainWindow* mainWnd = RiuMainWindow::instance();
if (!mainWnd) return;
RimProject* proj = RiaApplication::instance()->project();
if (!proj) return;
std::vector<RimCase*> projectCases;
proj->allCases(projectCases);
for (size_t i = 0; i < projectCases.size(); i++)
{
RimCase* cas = projectCases[i];
if (!cas) continue;
std::vector<Rim3dView*> views = cas->views();
for (size_t j = 0; j < views.size(); j++)
{
Rim3dView* riv = views[j];
if (riv && riv->viewer())
{
// Make sure all views are maximized for snapshotting
QMdiSubWindow* subWnd = mainWnd->findMdiSubWindow(riv->viewer()->layoutWidget());
if (subWnd)
{
subWnd->showMaximized();
}
// This size is set to match the regression test reference images
riv->viewer()->setFixedSize(RiaRegressionTestRunner::regressionDefaultImageSize());
}
}
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaRegressionTestRunner::resizeMaximizedPlotWindows()
{
RimProject* proj = RiaApplication::instance()->project();
if (!proj) return;
RiuMainPlotWindow* plotMainWindow = RiaApplication::instance()->mainPlotWindow();
if (!plotMainWindow) return;
std::vector<RimViewWindow*> viewWindows;
proj->mainPlotCollection()->descendantsIncludingThisOfType(viewWindows);
for (auto viewWindow : viewWindows)
{
if (viewWindow->isMdiWindow())
{
RimMdiWindowGeometry wndGeo = viewWindow->mdiWindowGeometry();
if (wndGeo.isMaximized)
{
QWidget* viewWidget = viewWindow->viewWidget();
if (viewWidget)
{
QMdiSubWindow* mdiWindow = plotMainWindow->findMdiSubWindow(viewWidget);
if (mdiWindow)
{
mdiWindow->showNormal();
viewWidget->resize(RiaRegressionTestRunner::regressionDefaultImageSize());
}
}
}
}
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
QSize RiaRegressionTestRunner::regressionDefaultImageSize()
{
return QSize(1000, 745);
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaRegressionTestRunner::executeRegressionTests()
{
RiaRegressionTest testConfig;
testConfig.readSettingsFromApplicationStore();
QString testPath = testConfig.regressionTestFolder();
QStringList testFilter = testConfig.testFilter().split(";", QString::SkipEmptyParts);
executeRegressionTests(testPath, testFilter);
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaRegressionTestRunner::executeRegressionTests(const QString& regressionTestPath, const QStringList& testFilter)
{
RiuMainWindow* mainWnd = RiuMainWindow::instance();
if (mainWnd)
{
mainWnd->hideAllDockWindows();
mainWnd->setDefaultWindowSize();
runRegressionTest(regressionTestPath, testFilter);
mainWnd->loadWinGeoAndDockToolBarLayout();
}
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
bool RiaRegressionTestRunner::isRunningRegressionTests() const
{
return m_runningRegressionTests;
}
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
void RiaRegressionTestRunner::updateRegressionTest(const QString& testRootPath)
{
// Find all sub folders
QDir testDir(testRootPath); // If string is empty it will end up as cwd
testDir.setFilter(QDir::Dirs);
QStringList dirNameFilter;
dirNameFilter.append(RegTestNames::testFolderFilter);
testDir.setNameFilters(dirNameFilter);
QFileInfoList folderList = testDir.entryInfoList();
for (int i = 0; i < folderList.size(); ++i)
{
QDir testCaseFolder(folderList[i].filePath());
QDir baseDir(testCaseFolder.filePath(RegTestNames::baseFolderName));
removeDirectoryWithContent(baseDir);
testCaseFolder.mkdir(RegTestNames::baseFolderName);
QDir genDir(testCaseFolder.filePath(RegTestNames::generatedFolderName));
QStringList imageFileNames = genDir.entryList();
for (int fIdx = 0; fIdx < imageFileNames.size(); ++fIdx)
{
QString fileName = imageFileNames[fIdx];
QFile::copy(genDir.filePath(fileName), baseDir.filePath(fileName));
}
}
}

55
ApplicationCode/Application/Tools/RiaRegressionTestRunner.h Normal file
View File

@ -0,0 +1,55 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2018 Statoil ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <QSize>
#include <QStringList>
class QDir;
//==================================================================================================
//
//==================================================================================================
class RiaRegressionTestRunner
{
public:
RiaRegressionTestRunner();
static RiaRegressionTestRunner* instance();
void executeRegressionTests(const QString& regressionTestPath, const QStringList& testFilter);
void executeRegressionTests();
bool isRunningRegressionTests() const;
static void updateRegressionTest(const QString& testRootPath);
private:
void runRegressionTest(const QString& testRootPath, const QStringList& testFilter);
static void removeDirectoryWithContent(QDir& dirToDelete);
static void regressionTestConfigureProject();
static void resizeMaximizedPlotWindows();
static QSize regressionDefaultImageSize();
private:
const QString m_rootPath;
const QStringList m_testFilter;
bool m_runningRegressionTests;
};

2
ApplicationCode/Application/Tools/RiaTextFileCompare.cpp
View File

@ -61,7 +61,7 @@ bool RiaTextFileCompare::runComparison(const QString& baseFolder, const QString&
}
// Run compare recursively with '-r'
QString args = "-r";
QString args = "-r -u";
QString completeCommand = QString("\"%1\" %2 %3 %4").arg(fullFilePath).arg(baseFolder).arg(generatedFolder).arg(args);

5
ApplicationCode/Commands/ApplicationCommands/RicLaunchRegressionTestsFeature.cpp
View File

@ -18,8 +18,7 @@
#include "RicLaunchRegressionTestsFeature.h"
#include "RiaApplication.h"
#include "RiaRegressionTest.h"
#include "RiaRegressionTestRunner.h"
#include "cafPdmSettings.h"
@ -41,7 +40,7 @@ bool RicLaunchRegressionTestsFeature::isCommandEnabled()
//--------------------------------------------------------------------------------------------------
void RicLaunchRegressionTestsFeature::onActionTriggered(bool isChecked)
{
RiaApplication::instance()->executeRegressionTests();
RiaRegressionTestRunner::instance()->executeRegressionTests();
}
//--------------------------------------------------------------------------------------------------

4
ApplicationCode/UserInterface/RiuMainWindow.cpp
View File

@ -24,6 +24,7 @@
#include "RiaBaseDefs.h"
#include "RiaPreferences.h"
#include "RiaRegressionTest.h"
#include "RiaRegressionTestRunner.h"
#include "Rim3dView.h"
#include "RimCellEdgeColors.h"
@ -1726,8 +1727,7 @@ void RiuMainWindow::slotShowRegressionTestDialog()
// Write preferences using QSettings and apply them to the application
regTestConfig.writeSettingsToApplicationStore();
RiaApplication* app = RiaApplication::instance();
app->executeRegressionTests();
RiaRegressionTestRunner::instance()->executeRegressionTests();
}
}

3
ApplicationCode/UserInterface/RiuViewer.cpp
View File

@ -23,6 +23,7 @@
#include "RiaApplication.h"
#include "RiaBaseDefs.h"
#include "RiaColorTools.h"
#include "RiaRegressionTestRunner.h"
#include "RimCase.h"
#include "RimProject.h"
@ -157,7 +158,7 @@ RiuViewer::RiuViewer(const QGLFormat& format, QWidget* parent)
m_viewerCommands = new RiuViewerCommands(this);
if (RiaApplication::instance()->isRunningRegressionTests())
if (RiaRegressionTestRunner::instance()->isRunningRegressionTests())
{
QFont regTestFont = m_infoLabel->font();
regTestFont.setPixelSize(11);