From e1f4585cb8e30023f9fd1708d503fc05562828f9 Mon Sep 17 00:00:00 2001 From: Vitaliy Urusovskij Date: Thu, 29 Oct 2020 00:11:01 +0300 Subject: [PATCH] Add `validate_test_case` fixture with using of `jsonschema`. Specify all required fields for test cases (#2821) --- .../.automation/tgl_test_config.yml | 8 ++++ tests/time_tests/test_runner/conftest.py | 44 +++++++++++++++++++ tests/time_tests/test_runner/requirements.txt | 3 +- tests/time_tests/test_runner/test_config.yml | 8 +++- tests/time_tests/test_runner/test_timetest.py | 3 +- 5 files changed, 63 insertions(+), 3 deletions(-) diff --git a/tests/time_tests/test_runner/.automation/tgl_test_config.yml b/tests/time_tests/test_runner/.automation/tgl_test_config.yml index 1ccbd2358bd..4c97c80f21c 100644 --- a/tests/time_tests/test_runner/.automation/tgl_test_config.yml +++ b/tests/time_tests/test_runner/.automation/tgl_test_config.yml @@ -2,47 +2,55 @@ name: CPU model: path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml + name: resnet-50-pytorch precision: FP16 framework: caffe2 - device: name: GPU model: path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml + name: resnet-50-pytorch precision: FP16 framework: caffe2 - device: name: CPU model: path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml + name: resnet-50-pytorch precision: FP16-INT8 framework: caffe2 - device: name: GPU model: path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml + name: resnet-50-pytorch precision: FP16-INT8 framework: caffe2 - device: name: CPU model: path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml + name: mobilenet-v2 precision: FP16 framework: caffe2 - device: name: GPU model: path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml + name: mobilenet-v2 precision: FP16 framework: caffe2 - device: name: CPU model: path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml + name: mobilenet-v2 precision: FP16-INT8 framework: caffe2 - device: name: GPU model: path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml + name: mobilenet-v2 precision: FP16-INT8 framework: caffe2 \ No newline at end of file diff --git a/tests/time_tests/test_runner/conftest.py b/tests/time_tests/test_runner/conftest.py index e7e7c1464ad..a2a0df40312 100644 --- a/tests/time_tests/test_runner/conftest.py +++ b/tests/time_tests/test_runner/conftest.py @@ -25,6 +25,7 @@ import hashlib import shutil import logging import tempfile +from jsonschema import validate, ValidationError from test_runner.utils import upload_timetest_data, \ DATABASE, DB_COLLECTIONS @@ -155,6 +156,45 @@ def test_info(request, pytestconfig): pytestconfig.session_info.append(request.node._request.test_info) +@pytest.fixture(scope="function") +def validate_test_case(request, test_info): + """Fixture for validating test case on correctness. + + Fixture checks current test case contains all fields required for + a correct work. To submit results to a database test case have + contain several additional properties. + """ + schema = { + "type": "object", + "properties": { + "device": { + "type": "object", + "properties": { + "name": {"type": "string"} + }}, + "model": { + "type": "object", + "properties": { + "path": {"type": "string"} + }}, + }, + } + if request.config.getoption("db_submit"): + # For submission data to a database some additional fields are required + schema["properties"]["model"]["properties"].update({ + "name": {"type": "string"}, + "precision": {"type": "string"}, + "framework": {"type": "string"} + }) + test_info["submit_to_db"] = True + try: + validate(instance=request.node.funcargs["instance"], schema=schema) + except ValidationError: + test_info["submit_to_db"] = False + raise + yield + + @pytest.fixture(scope="session", autouse=True) def prepare_tconf_with_refs(pytestconfig): """Fixture for preparing test config based on original test config @@ -220,6 +260,10 @@ def pytest_runtest_makereport(item, call): if not (run_id and db_url and db_collection): yield return + if not item._request.test_info["submit_to_db"]: + logging.error("Data won't be uploaded to a database on '{}' step".format(call.when)) + yield + return data = item.funcargs.copy() data["timetest"] = data.pop("executable").stem diff --git a/tests/time_tests/test_runner/requirements.txt b/tests/time_tests/test_runner/requirements.txt index 13426ece5e1..ae3ab36a70d 100644 --- a/tests/time_tests/test_runner/requirements.txt +++ b/tests/time_tests/test_runner/requirements.txt @@ -1,3 +1,4 @@ pytest==4.0.1 attrs==19.1.0 # required for pytest==4.0.1 to resolve compatibility issues -PyYAML==5.3.1 \ No newline at end of file +PyYAML==5.3.1 +jsonschema==3.2.0 \ No newline at end of file diff --git a/tests/time_tests/test_runner/test_config.yml b/tests/time_tests/test_runner/test_config.yml index bfcb5652646..3f7c82f316f 100644 --- a/tests/time_tests/test_runner/test_config.yml +++ b/tests/time_tests/test_runner/test_config.yml @@ -2,7 +2,13 @@ name: CPU model: path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model + name: alexnet + precision: FP32 + framework: caffe - device: name: GPU model: - path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml \ No newline at end of file + path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml + name: alexnet + precision: FP32 + framework: caffe diff --git a/tests/time_tests/test_runner/test_timetest.py b/tests/time_tests/test_runner/test_timetest.py index 9c1e11d6da1..a9aaf747c18 100644 --- a/tests/time_tests/test_runner/test_timetest.py +++ b/tests/time_tests/test_runner/test_timetest.py @@ -25,7 +25,7 @@ from test_runner.utils import expand_env_vars REFS_FACTOR = 1.2 # 120% -def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir): +def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir, validate_test_case): """Parameterized test. :param instance: test instance. Should not be changed during test run @@ -34,6 +34,7 @@ def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir :param cl_cache_dir: directory to store OpenCL cache :param test_info: custom `test_info` field of built-in `request` pytest fixture :param temp_dir: path to a temporary directory. Will be cleaned up after test run + :param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled """ # Prepare model to get model_path model_path = instance["model"].get("path")