diff --git a/compareECLFiles.cmake b/compareECLFiles.cmake
index 1c572b8d3..9b573aa6f 100644
--- a/compareECLFiles.cmake
+++ b/compareECLFiles.cmake
@@ -12,12 +12,12 @@ set(BASE_RESULT_PATH ${PROJECT_BINARY_DIR}/tests/results)
 # Input:
 #   - casename: basename (no extension)
 #
-macro (add_test_compareECLFiles casename filename)
+macro (add_test_compareECLFiles casename filename simulator)
 
-  set(RESULT_PATH ${BASE_RESULT_PATH}/${casename})
+  set(RESULT_PATH ${BASE_RESULT_PATH}/${casename}+${simulator})
   # Add test that runs flow and outputs the results to file
-  opm_add_test(compareECLFiles_${filename} NO_COMPILE
-               EXE_NAME flow
+  opm_add_test(compareECLFiles_${simulator}+${filename} NO_COMPILE
+               EXE_NAME ${simulator}
                DRIVER_ARGS ${OPM_DATA_ROOT}/${casename} ${RESULT_PATH}
                            ${CMAKE_BINARY_DIR}/bin
                            ${filename}
@@ -34,12 +34,12 @@ endmacro (add_test_compareECLFiles)
 # Input:
 #   - casename: basename (no extension)
 #
-macro (add_test_compareECLRestartFiles casename filename)
+macro (add_test_compareECLRestartFiles casename filename simulator)
 
   set(RESULT_PATH ${BASE_RESULT_PATH}/restart/${casename})
   # Add test that runs flow and outputs the results to file
-  opm_add_test(compareECLRestartFiles_${filename} NO_COMPILE
-               EXE_NAME flow
+  opm_add_test(compareECLRestartFiles_${simulator}+${filename} NO_COMPILE
+               EXE_NAME ${simulator}
                DRIVER_ARGS ${OPM_DATA_ROOT}/${casename} ${RESULT_PATH}
                            ${CMAKE_BINARY_DIR}/bin
                            ${filename}
@@ -54,14 +54,15 @@ if(NOT TARGET test-suite)
   add_custom_target(test-suite)
 endif()
 
+# Regression tests
 opm_set_test_driver(${PROJECT_SOURCE_DIR}/tests/run-regressionTest.sh "")
 
-add_test_compareECLFiles(spe1 SPE1CASE2)
-add_test_compareECLFiles(spe3 SPE3CASE1)
-add_test_compareECLFiles(spe9 SPE9_CP_SHORT)
+add_test_compareECLFiles(spe1 SPE1CASE2 flow)
+add_test_compareECLFiles(spe3 SPE3CASE1 flow)
+add_test_compareECLFiles(spe9 SPE9_CP_SHORT flow)
 
 # Restart tests
 opm_set_test_driver(${PROJECT_SOURCE_DIR}/tests/run-restart-regressionTest.sh "")
 
-add_test_compareECLRestartFiles(spe9 SPE9_CP_SHORT)
-add_test_compareECLRestartFiles(spe1 SPE1CASE2_ACTNUM)
+add_test_compareECLRestartFiles(spe1 SPE1CASE2_ACTNUM flow)
+add_test_compareECLRestartFiles(spe9 SPE9_CP_SHORT flow)