Add validate_test_case
fixture with using of jsonschema
. Specify all required fields for test cases (#2821)
This commit is contained in:
parent
347e1206d5
commit
e1f4585cb8
@ -2,47 +2,55 @@
|
||||
name: CPU
|
||||
model:
|
||||
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
|
||||
name: resnet-50-pytorch
|
||||
precision: FP16
|
||||
framework: caffe2
|
||||
- device:
|
||||
name: GPU
|
||||
model:
|
||||
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
|
||||
name: resnet-50-pytorch
|
||||
precision: FP16
|
||||
framework: caffe2
|
||||
- device:
|
||||
name: CPU
|
||||
model:
|
||||
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
|
||||
name: resnet-50-pytorch
|
||||
precision: FP16-INT8
|
||||
framework: caffe2
|
||||
- device:
|
||||
name: GPU
|
||||
model:
|
||||
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
|
||||
name: resnet-50-pytorch
|
||||
precision: FP16-INT8
|
||||
framework: caffe2
|
||||
- device:
|
||||
name: CPU
|
||||
model:
|
||||
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
|
||||
name: mobilenet-v2
|
||||
precision: FP16
|
||||
framework: caffe2
|
||||
- device:
|
||||
name: GPU
|
||||
model:
|
||||
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
|
||||
name: mobilenet-v2
|
||||
precision: FP16
|
||||
framework: caffe2
|
||||
- device:
|
||||
name: CPU
|
||||
model:
|
||||
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
|
||||
name: mobilenet-v2
|
||||
precision: FP16-INT8
|
||||
framework: caffe2
|
||||
- device:
|
||||
name: GPU
|
||||
model:
|
||||
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
|
||||
name: mobilenet-v2
|
||||
precision: FP16-INT8
|
||||
framework: caffe2
|
@ -25,6 +25,7 @@ import hashlib
|
||||
import shutil
|
||||
import logging
|
||||
import tempfile
|
||||
from jsonschema import validate, ValidationError
|
||||
|
||||
from test_runner.utils import upload_timetest_data, \
|
||||
DATABASE, DB_COLLECTIONS
|
||||
@ -155,6 +156,45 @@ def test_info(request, pytestconfig):
|
||||
pytestconfig.session_info.append(request.node._request.test_info)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def validate_test_case(request, test_info):
|
||||
"""Fixture for validating test case on correctness.
|
||||
|
||||
Fixture checks current test case contains all fields required for
|
||||
a correct work. To submit results to a database test case have
|
||||
contain several additional properties.
|
||||
"""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"device": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"}
|
||||
}},
|
||||
"model": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string"}
|
||||
}},
|
||||
},
|
||||
}
|
||||
if request.config.getoption("db_submit"):
|
||||
# For submission data to a database some additional fields are required
|
||||
schema["properties"]["model"]["properties"].update({
|
||||
"name": {"type": "string"},
|
||||
"precision": {"type": "string"},
|
||||
"framework": {"type": "string"}
|
||||
})
|
||||
test_info["submit_to_db"] = True
|
||||
try:
|
||||
validate(instance=request.node.funcargs["instance"], schema=schema)
|
||||
except ValidationError:
|
||||
test_info["submit_to_db"] = False
|
||||
raise
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def prepare_tconf_with_refs(pytestconfig):
|
||||
"""Fixture for preparing test config based on original test config
|
||||
@ -220,6 +260,10 @@ def pytest_runtest_makereport(item, call):
|
||||
if not (run_id and db_url and db_collection):
|
||||
yield
|
||||
return
|
||||
if not item._request.test_info["submit_to_db"]:
|
||||
logging.error("Data won't be uploaded to a database on '{}' step".format(call.when))
|
||||
yield
|
||||
return
|
||||
|
||||
data = item.funcargs.copy()
|
||||
data["timetest"] = data.pop("executable").stem
|
||||
|
@ -1,3 +1,4 @@
|
||||
pytest==4.0.1
|
||||
attrs==19.1.0 # required for pytest==4.0.1 to resolve compatibility issues
|
||||
PyYAML==5.3.1
|
||||
PyYAML==5.3.1
|
||||
jsonschema==3.2.0
|
@ -2,7 +2,13 @@
|
||||
name: CPU
|
||||
model:
|
||||
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model
|
||||
name: alexnet
|
||||
precision: FP32
|
||||
framework: caffe
|
||||
- device:
|
||||
name: GPU
|
||||
model:
|
||||
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
|
||||
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
|
||||
name: alexnet
|
||||
precision: FP32
|
||||
framework: caffe
|
||||
|
@ -25,7 +25,7 @@ from test_runner.utils import expand_env_vars
|
||||
REFS_FACTOR = 1.2 # 120%
|
||||
|
||||
|
||||
def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir):
|
||||
def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir, validate_test_case):
|
||||
"""Parameterized test.
|
||||
|
||||
:param instance: test instance. Should not be changed during test run
|
||||
@ -34,6 +34,7 @@ def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir
|
||||
:param cl_cache_dir: directory to store OpenCL cache
|
||||
:param test_info: custom `test_info` field of built-in `request` pytest fixture
|
||||
:param temp_dir: path to a temporary directory. Will be cleaned up after test run
|
||||
:param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled
|
||||
"""
|
||||
# Prepare model to get model_path
|
||||
model_path = instance["model"].get("path")
|
||||
|
Loading…
Reference in New Issue
Block a user