Added a regression test system to ResInsight.

Based on image comparison. Command line invocation.
p4#: 20433

ApplicationCode/Application/RIApplication.cpp

@@ -53,6 +53,8 @@
 #include "cafUiProcess.h"
 
 #include "RimUiTreeModelPdm.h"
+#include "RiaImageCompareReporter.h"
+#include "RiaImageFileCompare.h"
 
 namespace caf
 {
@@ -66,6 +68,18 @@ void AppEnum< RIApplication::RINavigationPolicy >::setUp()
 }
 
 
+namespace RegTestData
+{
+    const QString generatedFolderName   = "RegTestGeneratedImages";
+    const QString diffFolderName        = "RegTestDiffImages";
+    const QString baseFolderName        = "RegTestBaseImages";
+    const QString testProjectName       = "RegressionTest.rip";
+    const QString testFolderFilter      = "TestCase*";
+    const QString imageCompareExeName   = "compare";
+    const QString reportFileName        = "ResInsightRegressionTestReport.html";
+};
+
+
 //==================================================================================================
 ///
 /// \class RIApplication
@@ -214,6 +228,7 @@ bool RIApplication::loadProject(const QString& projectFileName)
 
     m_project->fileName = projectFileName;
     m_project->readFile();
+    m_project->fileName = projectFileName; // Make sure we overwrite the old filename read from the project file 
 
     if (m_project->projectFileVersionString().isEmpty())
     {
@@ -602,12 +617,23 @@ bool RIApplication::parseArguments()
     bool openLatestProject = false;
     QString projectFilename;
     QStringList caseNames;
+    QString regressionTestPath;
+
+    enum ArgumentParsingType
+    {
+        PARSE_PROJECT_FILE_NAME,
+        PARSE_CASE_NAMES,
+        PARSE_START_DIR,
+        PARSE_REGRESSION_TEST_PATH,
+        PARSING_NONE
+    };
+
+    ArgumentParsingType argumentParsingType = PARSING_NONE;
 
-    bool isParsingProjectFile = false;
-    bool isParsingCaseNames = false;
-    bool isParsingStartDir = false;
     bool showHelp = false;
     bool isSaveSnapshotsForAllViews = false;
+    bool isRunRegressionTest = false;
+    bool isUpdateRegressionTest = false;
     
     int i;
     for (i = 1; i < arguments.size(); i++)
@@ -628,25 +654,19 @@ bool RIApplication::parseArguments()
         }
         else if (arg.toLower() == "-project")
         {
-            isParsingCaseNames = false;
-            isParsingProjectFile = true;
-            isParsingStartDir = false;
+            argumentParsingType = PARSE_PROJECT_FILE_NAME;
 
             foundKnownOption = true;
         }
         else if (arg.toLower() == "-case")
         {
-            isParsingCaseNames = true;
-            isParsingProjectFile = false;
-            isParsingStartDir = false;
+            argumentParsingType = PARSE_CASE_NAMES;
 
             foundKnownOption = true;
         }
         else if (arg.toLower() == "-startdir")
         {
-            isParsingCaseNames = false;
-            isParsingProjectFile = false;
-            isParsingStartDir = true;
+            argumentParsingType = PARSE_START_DIR;
 
             foundKnownOption = true;
         }
@@ -656,22 +676,48 @@ bool RIApplication::parseArguments()
 
             foundKnownOption = true;
         }
+        else if (arg.toLower() == "-regressiontest")
+        {
+            isRunRegressionTest = true; 
 
+            argumentParsingType = PARSE_REGRESSION_TEST_PATH;
+
+            foundKnownOption = true;
+        }
+        else if (arg.toLower() == "-updateregressiontestbase")
+        {
+            isUpdateRegressionTest = true; 
+
+            argumentParsingType = PARSE_REGRESSION_TEST_PATH;
+
+            foundKnownOption = true;
+        }
+        
         if (!foundKnownOption)
         {
-            if (isParsingProjectFile && QFile::exists(arg))
+            switch (argumentParsingType)
             {
-                projectFilename = arg;
-            }
+            case PARSE_PROJECT_FILE_NAME:
+                if (QFile::exists(arg))
+                {
+                    projectFilename = arg;
+                }
+                break;
+            case PARSE_CASE_NAMES:
+                {
+                    caseNames.append(arg);
+                }
+                break;
 
-            if (isParsingCaseNames)
-            {
-                caseNames.append(arg);
-            }
-
-            if (isParsingStartDir)
-            {
-                m_startupDefaultDirectory = arg;
+            case PARSE_START_DIR:
+                {
+                    m_startupDefaultDirectory = arg;
+                }
+                break;
+            case PARSE_REGRESSION_TEST_PATH:
+                {
+                   regressionTestPath = arg; 
+                }
             }
         }
     }
@@ -682,19 +728,33 @@ bool RIApplication::parseArguments()
         helpText += "Copyright Statoil ASA, Ceetron AS 2011, 2012\n\n";
         
         helpText +=
-        "\nParameter            Description\n"
+        "\nParameter              Description\n"
         "-----------------------------------------------------------------\n"
-        "-last                Open last used project\n"
-        "-project <filename>  Open project file <filename>\n"
-        "-case <casename>     Open Eclipse case <casename>\n"
-        "                     (do not include .GRID/.EGRID)\n"
-        "-startdir            The default directory for open/save commands\n"
+        "-last                    Open last used project\n"
         "\n"
-        "-savesnapshots       Save snapshot of all views to 'snapshots' folder in project file folder\n"
-        "                     Application closes after snapshots are written to file\n"
+        "-project <filename>      Open project file <filename>\n"
         "\n"
-        "-help                \n"
-        "-?                   Displays help text\n"
+        "-case <casename>         Open Eclipse case <casename>\n"
+        "                         (do not include .GRID/.EGRID)\n"
+        "\n"
+        "-startdir                The default directory for open/save commands\n"
+        "\n"                      
+        "-savesnapshots           Save snapshot of all views to 'snapshots' folder in project file folder\n"
+        "                         Application closes after snapshots are written to file\n"
+        "\n"
+        "-regressiontest <folder> Run a regression test on all sub-folders of the given folder: \n"
+        "                         RegressionTest.rip files in the sub-folders will be opened and \n"
+        "                         snapshots of all the views is written to the sub-sub-folder RegTestGeneratedImages. \n"
+        "                         Then difference images is generated in the sub-sub-folder RegTestDiffImages based \n"
+        "                         on the images in sub-sub-folder RegTestBaseImages.\n"
+        "                         The results are presented in ResInsightRegressionTestReport.html that is\n"
+        "                         written in the given folder.\n"
+        "\n"
+        "-updateregressiontestbase <folder> For all sub-folders of the given folder: \n"
+        "                         Copy the images in the sub-sub-folder RegTestGeneratedImages to the sub-sub-folder\n" 
+        "                         RegTestBaseImages after deleting RegTestBaseImages completely.\n"
+        "\n"
+        "-help, -?                Displays help text\n"
         "-----------------------------------------------------------------";
 
         fprintf(stdout, "%s\n", helpText.toAscii().data());
@@ -703,6 +763,20 @@ bool RIApplication::parseArguments()
         return false;
     }
 
+    if (isRunRegressionTest)
+    {
+        runRegressionTest(regressionTestPath);
+
+        return false;
+    }
+
+    if (isUpdateRegressionTest)
+    {
+        updateRegressionTest(regressionTestPath);
+
+        return false;
+    }
+
     if (openLatestProject)
     {
         loadLastUsedProject();
@@ -737,12 +811,13 @@ bool RIApplication::parseArguments()
 
     if (m_project.notNull() && !m_project->fileName().isEmpty() && isSaveSnapshotsForAllViews)
     {
-        saveSnapshotForAllViews();
+        saveSnapshotForAllViews("snapshots");
 
         // Returning false will exit the application
         return false;
     }
 
+
     return true;
 }
 
@@ -1031,7 +1106,7 @@ void RIApplication::copySnapshotToClipboard()
 //--------------------------------------------------------------------------------------------------
 /// 
 //--------------------------------------------------------------------------------------------------
-void RIApplication::saveSnapshotForAllViews()
+void RIApplication::saveSnapshotForAllViews(const QString& snapshotFolderName)
 {
     RIMainWindow* mainWnd = RIMainWindow::instance();
     if (!mainWnd) return;
@@ -1042,8 +1117,7 @@ void RIApplication::saveSnapshotForAllViews()
 
     QFileInfo fi(m_project->fileName());
     QDir projectDir(fi.absolutePath());
-
-    QString snapshotFolderName = "snapshots";
+    
     if (!projectDir.exists(snapshotFolderName))
     {
         if (!projectDir.mkdir(snapshotFolderName)) return;
@@ -1073,9 +1147,141 @@ void RIApplication::saveSnapshotForAllViews()
 
                 QString fileName = ri->caseName() + "-" + riv->name();
 
-                QString absoluteFileName = caf::Utils::constructFullFileName(snapshotPath, fileName, ".PNG");
+                QString absoluteFileName = caf::Utils::constructFullFileName(snapshotPath, fileName, ".png");
                 saveSnapshotAs(absoluteFileName);
             }
         }
     }
 }
+
+void removeDirectoryWithContent(QDir dirToDelete )
+{
+    QStringList files = dirToDelete.entryList();
+    for (int fIdx = 0; fIdx < files.size(); ++fIdx)
+    {
+        dirToDelete.remove(files[fIdx]);
+    }
+    dirToDelete.rmdir(".");
+}
+
+
+//--------------------------------------------------------------------------------------------------
+/// 
+//--------------------------------------------------------------------------------------------------
+void RIApplication::runRegressionTest(const QString& testRootPath)
+{
+    QString generatedFolderName = RegTestData::generatedFolderName;
+    QString diffFolderName      = RegTestData::diffFolderName;    
+    QString baseFolderName      = RegTestData::baseFolderName;    
+    QString regTestProjectName  = RegTestData::testProjectName; 
+    QString regTestFolderFilter = RegTestData::testFolderFilter;
+
+    // Find all sub folders
+    
+    QDir testDir(testRootPath); // If string is empty it will end up as cwd
+    testDir.setFilter(QDir::Dirs);
+    QStringList dirNameFilter;
+    dirNameFilter.append(regTestFolderFilter);
+    testDir.setNameFilters(dirNameFilter);
+
+    QFileInfoList folderList = testDir.entryInfoList();
+
+    // delete diff and generated images
+
+
+    for (int i = 0; i < folderList.size(); ++i)
+    {
+        QDir testCaseFolder(folderList[i].filePath());
+
+        QDir genDir(testCaseFolder.filePath(generatedFolderName));
+        removeDirectoryWithContent(genDir);
+
+        QDir diffDir(testCaseFolder.filePath(diffFolderName));
+        removeDirectoryWithContent(diffDir);
+
+        QDir baseDir(testCaseFolder.filePath(baseFolderName));
+    }
+
+    // Generate html report
+
+    RiaImageCompareReporter imageCompareReporter;
+
+    for (int dirIdx = 0; dirIdx < folderList.size(); ++dirIdx)
+    {
+        QDir testCaseFolder(folderList[dirIdx].filePath());
+
+        QString testFolderName = testCaseFolder.dirName();
+        QString reportBaseFolderName       = testCaseFolder.filePath(baseFolderName);
+        QString reportGeneratedFolderName  = testCaseFolder.filePath(generatedFolderName);
+        QString reportDiffFolderName       = testCaseFolder.filePath(diffFolderName);
+
+        imageCompareReporter.addImageDirectoryComparisonSet(testFolderName.toStdString(), reportBaseFolderName.toStdString(), reportGeneratedFolderName.toStdString(), reportDiffFolderName.toStdString());
+    }
+
+    imageCompareReporter.generateHTMLReport(testDir.filePath(RegTestData::reportFileName).toStdString());
+
+    // Generate diff images
+
+    for (int dirIdx = 0; dirIdx < folderList.size(); ++dirIdx)
+    {
+        QDir testCaseFolder(folderList[dirIdx].filePath());
+        if (testCaseFolder.exists(regTestProjectName))
+        {
+             loadProject(testCaseFolder.filePath(regTestProjectName));
+             saveSnapshotForAllViews(generatedFolderName);
+
+             QDir baseDir(testCaseFolder.filePath(baseFolderName));
+             QDir genDir(testCaseFolder.filePath(generatedFolderName));
+             QDir diffDir(testCaseFolder.filePath(diffFolderName));
+             if (!diffDir.exists()) testCaseFolder.mkdir(diffFolderName);
+             baseDir.setFilter(QDir::Files);
+             QStringList baseImageFileNames = baseDir.entryList();
+
+             for (int fIdx = 0; fIdx < baseImageFileNames.size(); ++fIdx)
+             {
+                 QString fileName = baseImageFileNames[fIdx];
+                 RiaImageFileCompare imgComparator(RegTestData::imageCompareExeName);
+                 bool ok = imgComparator.runComparison(genDir.filePath(fileName), baseDir.filePath(fileName), diffDir.filePath(fileName));
+                 if (!ok)
+                 {
+                    qDebug() << "Error comparing :" << imgComparator.errorMessage() << "\n" << imgComparator.errorDetails();
+                 }
+             }
+        }
+    }
+}
+
+//--------------------------------------------------------------------------------------------------
+/// 
+//--------------------------------------------------------------------------------------------------
+void RIApplication::updateRegressionTest(const QString& testRootPath)
+{
+    // Find all sub folders
+
+    QDir testDir(testRootPath); // If string is empty it will end up as cwd
+    testDir.setFilter(QDir::Dirs);
+    QStringList dirNameFilter;
+    dirNameFilter.append(RegTestData::testFolderFilter);
+    testDir.setNameFilters(dirNameFilter);
+
+    QFileInfoList folderList = testDir.entryInfoList();
+
+    for (int i = 0; i < folderList.size(); ++i)
+    {
+        QDir testCaseFolder(folderList[i].filePath());
+
+        QDir baseDir(testCaseFolder.filePath(RegTestData::baseFolderName));
+        removeDirectoryWithContent(baseDir);
+        testCaseFolder.mkdir(RegTestData::baseFolderName);
+
+        QDir genDir(testCaseFolder.filePath(RegTestData::generatedFolderName));
+
+        QStringList imageFileNames = genDir.entryList();
+
+        for (int fIdx = 0; fIdx < imageFileNames.size(); ++fIdx)
+        {
+            QString fileName = imageFileNames[fIdx];
+            QFile::copy(genDir.filePath(fileName), baseDir.filePath(fileName));
+        }
+    }
+}