Add validate_test_case fixture with using of jsonschema. Specify all required fields for test cases (#2821)

tests/time_tests/test_runner/.automation/tgl_test_config.yml

@@ -2,47 +2,55 @@
     name: CPU
   model:
     path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
+    name: resnet-50-pytorch
     precision: FP16
     framework: caffe2
 - device:
     name: GPU
   model:
     path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
+    name: resnet-50-pytorch
     precision: FP16
     framework: caffe2
 - device:
     name: CPU
   model:
     path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
+    name: resnet-50-pytorch
     precision: FP16-INT8
     framework: caffe2
 - device:
     name: GPU
   model:
     path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
+    name: resnet-50-pytorch
     precision: FP16-INT8
     framework: caffe2
 - device:
     name: CPU
   model:
     path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
+    name: mobilenet-v2
     precision: FP16
     framework: caffe2
 - device:
     name: GPU
   model:
     path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
+    name: mobilenet-v2
     precision: FP16
     framework: caffe2
 - device:
     name: CPU
   model:
     path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
+    name: mobilenet-v2
     precision: FP16-INT8
     framework: caffe2
 - device:
     name: GPU
   model:
     path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
+    name: mobilenet-v2
     precision: FP16-INT8
     framework: caffe2

tests/time_tests/test_runner/conftest.py

@@ -25,6 +25,7 @@ import hashlib
 import shutil
 import logging
 import tempfile
+from jsonschema import validate, ValidationError
 
 from test_runner.utils import upload_timetest_data, \
     DATABASE, DB_COLLECTIONS
@@ -155,6 +156,45 @@ def test_info(request, pytestconfig):
     pytestconfig.session_info.append(request.node._request.test_info)
 
 
+@pytest.fixture(scope="function")
+def validate_test_case(request, test_info):
+    """Fixture for validating test case on correctness.
+
+    Fixture checks current test case contains all fields required for
+    a correct work. To submit results to a database test case have
+    contain several additional properties.
+    """
+    schema = {
+        "type": "object",
+        "properties": {
+            "device": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"}
+                }},
+            "model": {
+                "type": "object",
+                "properties": {
+                    "path": {"type": "string"}
+                }},
+        },
+    }
+    if request.config.getoption("db_submit"):
+        # For submission data to a database some additional fields are required
+        schema["properties"]["model"]["properties"].update({
+            "name": {"type": "string"},
+            "precision": {"type": "string"},
+            "framework": {"type": "string"}
+        })
+    test_info["submit_to_db"] = True
+    try:
+        validate(instance=request.node.funcargs["instance"], schema=schema)
+    except ValidationError:
+        test_info["submit_to_db"] = False
+        raise
+    yield
+
+
 @pytest.fixture(scope="session", autouse=True)
 def prepare_tconf_with_refs(pytestconfig):
     """Fixture for preparing test config based on original test config
@@ -220,6 +260,10 @@ def pytest_runtest_makereport(item, call):
     if not (run_id and db_url and db_collection):
         yield
         return
+    if not item._request.test_info["submit_to_db"]:
+        logging.error("Data won't be uploaded to a database on '{}' step".format(call.when))
+        yield
+        return
 
     data = item.funcargs.copy()
     data["timetest"] = data.pop("executable").stem

tests/time_tests/test_runner/requirements.txt

@@ -1,3 +1,4 @@
 pytest==4.0.1
 attrs==19.1.0   # required for pytest==4.0.1 to resolve compatibility issues
-PyYAML==5.3.1
+PyYAML==5.3.1
+jsonschema==3.2.0

tests/time_tests/test_runner/test_config.yml

@@ -2,7 +2,13 @@
     name: CPU
   model:
     path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model
+    name: alexnet
+    precision: FP32
+    framework: caffe
 - device:
     name: GPU
   model:
-    path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
+    path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
+    name: alexnet
+    precision: FP32
+    framework: caffe

tests/time_tests/test_runner/test_timetest.py

@@ -25,7 +25,7 @@ from test_runner.utils import expand_env_vars
 REFS_FACTOR = 1.2      # 120%
 
 
-def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir):
+def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir, validate_test_case):
     """Parameterized test.
 
     :param instance: test instance. Should not be changed during test run
@@ -34,6 +34,7 @@ def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir
     :param cl_cache_dir: directory to store OpenCL cache
     :param test_info: custom `test_info` field of built-in `request` pytest fixture
     :param temp_dir: path to a temporary directory. Will be cleaned up after test run
+    :param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled
     """
     # Prepare model to get model_path
     model_path = instance["model"].get("path")