Add validate_test_case fixture with using of jsonschema. Specify all required fields for test cases (#2821)

This commit is contained in:
Vitaliy Urusovskij 2020-10-29 00:11:01 +03:00 committed by GitHub
parent 347e1206d5
commit e1f4585cb8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 63 additions and 3 deletions

View File

@ -2,47 +2,55 @@
name: CPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
name: resnet-50-pytorch
precision: FP16
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
name: resnet-50-pytorch
precision: FP16
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
name: resnet-50-pytorch
precision: FP16-INT8
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
name: resnet-50-pytorch
precision: FP16-INT8
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
name: mobilenet-v2
precision: FP16
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
name: mobilenet-v2
precision: FP16
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
name: mobilenet-v2
precision: FP16-INT8
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
name: mobilenet-v2
precision: FP16-INT8
framework: caffe2

View File

@ -25,6 +25,7 @@ import hashlib
import shutil
import logging
import tempfile
from jsonschema import validate, ValidationError
from test_runner.utils import upload_timetest_data, \
DATABASE, DB_COLLECTIONS
@ -155,6 +156,45 @@ def test_info(request, pytestconfig):
pytestconfig.session_info.append(request.node._request.test_info)
@pytest.fixture(scope="function")
def validate_test_case(request, test_info):
"""Fixture for validating test case on correctness.
Fixture checks current test case contains all fields required for
a correct work. To submit results to a database test case have
contain several additional properties.
"""
schema = {
"type": "object",
"properties": {
"device": {
"type": "object",
"properties": {
"name": {"type": "string"}
}},
"model": {
"type": "object",
"properties": {
"path": {"type": "string"}
}},
},
}
if request.config.getoption("db_submit"):
# For submission data to a database some additional fields are required
schema["properties"]["model"]["properties"].update({
"name": {"type": "string"},
"precision": {"type": "string"},
"framework": {"type": "string"}
})
test_info["submit_to_db"] = True
try:
validate(instance=request.node.funcargs["instance"], schema=schema)
except ValidationError:
test_info["submit_to_db"] = False
raise
yield
@pytest.fixture(scope="session", autouse=True)
def prepare_tconf_with_refs(pytestconfig):
"""Fixture for preparing test config based on original test config
@ -220,6 +260,10 @@ def pytest_runtest_makereport(item, call):
if not (run_id and db_url and db_collection):
yield
return
if not item._request.test_info["submit_to_db"]:
logging.error("Data won't be uploaded to a database on '{}' step".format(call.when))
yield
return
data = item.funcargs.copy()
data["timetest"] = data.pop("executable").stem

View File

@ -1,3 +1,4 @@
pytest==4.0.1
attrs==19.1.0 # required for pytest==4.0.1 to resolve compatibility issues
PyYAML==5.3.1
PyYAML==5.3.1
jsonschema==3.2.0

View File

@ -2,7 +2,13 @@
name: CPU
model:
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model
name: alexnet
precision: FP32
framework: caffe
- device:
name: GPU
model:
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
name: alexnet
precision: FP32
framework: caffe

View File

@ -25,7 +25,7 @@ from test_runner.utils import expand_env_vars
REFS_FACTOR = 1.2 # 120%
def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir):
def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir, validate_test_case):
"""Parameterized test.
:param instance: test instance. Should not be changed during test run
@ -34,6 +34,7 @@ def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir
:param cl_cache_dir: directory to store OpenCL cache
:param test_info: custom `test_info` field of built-in `request` pytest fixture
:param temp_dir: path to a temporary directory. Will be cleaned up after test run
:param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled
"""
# Prepare model to get model_path
model_path = instance["model"].get("path")