Add "cl_cache" preparation in test_timetest.py (#2490)

tests/time_tests/scripts/run_timetest.py

@@ -61,7 +61,7 @@ def read_stats(stats_path, stats: dict):
 def aggregate_stats(stats: dict):
     """Aggregate provided statistics"""
     return {step_name: {"avg": statistics.mean(duration_list),
-                        "stdev": statistics.stdev(duration_list)}
+                        "stdev": statistics.stdev(duration_list) if len(duration_list) > 1 else 0}
             for step_name, duration_list in stats.items()}
 
 

tests/time_tests/test_runner/conftest.py

@@ -22,6 +22,7 @@ from pathlib import Path
 import yaml
 import hashlib
 from copy import deepcopy
+import shutil
 
 from test_runner.utils import upload_timetest_data, \
     DATABASE, DB_COLLECTIONS
@@ -104,6 +105,22 @@ def niter(request):
 # -------------------- CLI options --------------------
 
 
+@pytest.fixture(scope="function")
+def cl_cache_dir(pytestconfig):
+    """Generate directory to save OpenCL cache before test run and clean up after run.
+
+    Folder `cl_cache` should be created in a directory where tests were run. In this case
+    cache will be saved correctly. This behaviour is OS independent.
+    More: https://github.com/intel/compute-runtime/blob/master/opencl/doc/FAQ.md#how-can-cl_cache-be-enabled
+    """
+    cl_cache_dir = pytestconfig.invocation_dir / "cl_cache"
+    if cl_cache_dir.exists():
+        shutil.rmtree(cl_cache_dir)
+    cl_cache_dir.mkdir()
+    yield cl_cache_dir
+    shutil.rmtree(cl_cache_dir)
+
+
 def pytest_generate_tests(metafunc):
     """Pytest hook for test generation.
 

tests/time_tests/test_runner/test_timetest.py

@@ -16,6 +16,7 @@ Options[*]:
 
 from pathlib import Path
 import logging
+import os
 
 from scripts.run_timetest import run_timetest
 from test_runner.utils import expand_env_vars
@@ -23,7 +24,7 @@ from test_runner.utils import expand_env_vars
 REFS_FACTOR = 1.2      # 120%
 
 
-def test_timetest(instance, executable, niter):
+def test_timetest(instance, executable, niter, cl_cache_dir):
     """Parameterized test.
 
     :param instance: test instance
@@ -41,6 +42,14 @@ def test_timetest(instance, executable, niter):
         "device": instance["device"]["name"],
         "niter": niter
     }
+    if exe_args["device"] == "GPU":
+        # Generate cl_cache via additional timetest run
+        _exe_args = exe_args.copy()
+        _exe_args["niter"] = 1
+        logging.info("Run timetest once to generate cl_cache to {}".format(cl_cache_dir))
+        run_timetest(_exe_args, log=logging)
+        assert os.listdir(cl_cache_dir), "cl_cache isn't generated"
+
     retcode, aggr_stats = run_timetest(exe_args, log=logging)
     assert retcode == 0, "Run of executable failed"