IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Moved memory tests to OV API 2.0 (#9924)

Browse Source 
* Moved memory tests to OV API 2.0

* Added configs for OV api 2, updated configs for api 1

* Commented several models in configs (no such models on omz)

* Updated fillTensors

* Fix to get network inputs

* Updated fillTensors and configs
This commit is contained in:
Lidia Toropova 2022-02-01 14:36:05 +03:00 committed by GitHub
parent 8892b7b327
commit 2ce7becc6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 6538 additions and 6885 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/lib/src/common_utils.cpp
View File

@ -85,4 +85,4 @@ void fillTensors(ov::InferRequest &infer_request, const std::vector<ov::Output<o
}
infer_request.set_input_tensor(i, input_tensor);
}
}
}

1313
tests/memory_tests/.automation/nightly_configs/desktop_test_api_2_config.yml Normal file
View File

File diff suppressed because it is too large Load Diff

1473
tests/memory_tests/.automation/nightly_configs/desktop_test_config.yml
View File

File diff suppressed because it is too large Load Diff

656
tests/memory_tests/.automation/nightly_configs/myriad_test_api_2_config.yml Normal file
View File

@ -0,0 +1,656 @@
# Models with precision: FP32 -- device: MYRIAD
- device:
name: MYRIAD
model:
name: mobilenet-v2-1.4-224
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: brain-tumor-segmentation-0001
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: faster_rcnn_inception_resnet_v2_atrous_coco
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-inception
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: efficientnet-b0
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-v1-1.0-224-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd_mobilenet_v1_coco
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v3
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: alexnet
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v4-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd300
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: vgg19
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v1
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v3-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mtcnn-o
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v1-tiny-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v1-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tiny-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd512
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: brain-tumor-segmentation-0002
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: Sphereface
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v2
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctdet_coco_dlav0_512
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctpn
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: i3d-rgb-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-v2
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-ssd
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnext-50
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: caffenet
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mtcnn-r
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnet-50
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: densenet-121
precision: FP32
source: omz
references: {}
# Models with precision: FP16 -- device: MYRIAD
- device:
name: MYRIAD
model:
name: mobilenet-v2-1.4-224
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: brain-tumor-segmentation-0001
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: faster_rcnn_inception_resnet_v2_atrous_coco
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-inception
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: efficientnet-b0
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-v1-1.0-224-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd_mobilenet_v1_coco
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v3
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: alexnet
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v4-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd300
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: vgg19
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v1
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v3-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mtcnn-o
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v1-tiny-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v1-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tiny-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd512
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: brain-tumor-segmentation-0002
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: Sphereface
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v2
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctdet_coco_dlav0_512
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctpn
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: i3d-rgb-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-v2
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-ssd
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnext-50
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: caffenet
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mtcnn-r
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnet-50
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: densenet-121
precision: FP16
source: omz
references: {}
# Models with precision: FP16-INT8 -- device: MYRIAD
- device:
name: MYRIAD
model:
name: vehicle-attributes-recognition-barrier-0039
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-action-recognition-0006
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-vehicle-bike-detection-crossroad-1016
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: age-gender-recognition-retail-0013
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: vehicle-detection-adas-0002
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: image-retrieval-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-retail-0002
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-attributes-recognition-crossroad-0230
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: action-recognition-0001-decoder
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-action-recognition-teacher-0002
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-vehicle-bike-detection-crossroad-0078
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: face-detection-adas-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: unet-camvid-onnx-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: human-pose-estimation-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: faster-rcnn-resnet101-coco-sparse-60-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: action-recognition-0001-encoder
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-ava-sparse-35-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-raisinghand-recognition-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-asl-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tiny-ava-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: license-plate-recognition-barrier-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-retail-0013
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: single-image-super-resolution-1032
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: landmarks-regression-retail-0009
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: driver-action-recognition-adas-0002-decoder
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-ava-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-action-recognition-0005
precision: FP16-INT8
source: omz
references: {}

732
tests/memory_tests/.automation/nightly_configs/myriad_test_config.yml
View File

@ -1,60 +1,4 @@
# Models with precision: FP32 -- device: MYRIAD
- device:
name: MYRIAD
model:
name: mobilenet-v2-1.4-224
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: brain-tumor-segmentation-0001
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: faster_rcnn_inception_resnet_v2_atrous_coco
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-inception
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: efficientnet-b0
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-v1-1.0-224-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mask_rcnn_resnet101_atrous_coco
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
@ -62,679 +6,3 @@
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnet-152
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: octave-resnext-50-0.25
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v3
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd_mobilenet_v2_coco
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: alexnet
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v4-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd300
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: vgg19
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctdet_coco_dlav0_384
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v1
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v3-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mtcnn-o
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v1-tiny-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v1-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tiny-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd512
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: densenet-169
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: brain-tumor-segmentation-0002
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: Sphereface
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v2
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctdet_coco_dlav0_512
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctpn
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: i3d-rgb-tf
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-v2
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-ssd
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnext-50
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: caffenet
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mtcnn-r
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnet-50
precision: FP32
source: omz
references: {}
- device:
name: MYRIAD
model:
name: densenet-121
precision: FP32
source: omz
references: {}
# Models with precision: FP16 -- device: MYRIAD
- device:
name: MYRIAD
model:
name: mobilenet-v2-1.4-224
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: brain-tumor-segmentation-0001
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: faster_rcnn_inception_resnet_v2_atrous_coco
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-inception
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: efficientnet-b0
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-v1-1.0-224-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mask_rcnn_resnet101_atrous_coco
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd_mobilenet_v1_coco
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnet-152
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: octave-resnext-50-0.25
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v3
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd_mobilenet_v2_coco
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: alexnet
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v4-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd300
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: vgg19
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctdet_coco_dlav0_384
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v1
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v3-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mtcnn-o
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v1-tiny-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v1-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tiny-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ssd512
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: densenet-169
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: brain-tumor-segmentation-0002
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: Sphereface
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: googlenet-v2
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctdet_coco_dlav0_512
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: ctpn
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: i3d-rgb-tf
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-v2
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mobilenet-ssd
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnext-50
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: caffenet
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: mtcnn-r
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: se-resnet-50
precision: FP16
source: omz
references: {}
- device:
name: MYRIAD
model:
name: densenet-121
precision: FP16
source: omz
references: {}
# Models with precision: FP16-INT8 -- device: MYRIAD
- device:
name: MYRIAD
model:
name: vehicle-attributes-recognition-barrier-0039
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-action-recognition-0006
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-vehicle-bike-detection-crossroad-1016
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: age-gender-recognition-retail-0013
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: vehicle-detection-adas-0002
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: image-retrieval-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-retail-0002
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-attributes-recognition-crossroad-0230
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: action-recognition-0001-decoder
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-action-recognition-teacher-0002
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-vehicle-bike-detection-crossroad-0078
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: face-detection-adas-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: unet-camvid-onnx-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: human-pose-estimation-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: faster-rcnn-resnet101-coco-sparse-60-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: action-recognition-0001-encoder
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-ava-sparse-35-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-raisinghand-recognition-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-asl-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-tiny-ava-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: license-plate-recognition-barrier-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-retail-0013
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: single-image-super-resolution-1032
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: landmarks-regression-retail-0009
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: driver-action-recognition-adas-0002-decoder
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: yolo-v2-ava-0001
precision: FP16-INT8
source: omz
references: {}
- device:
name: MYRIAD
model:
name: person-detection-action-recognition-0005
precision: FP16-INT8
source: omz
references: {}

4267
tests/memory_tests/.automation/weekly_configs/desktop_test_api_2_config.yml Normal file
View File

File diff suppressed because it is too large Load Diff

4457
tests/memory_tests/.automation/weekly_configs/desktop_test_config.yml
View File

File diff suppressed because it is too large Load Diff

3
tests/memory_tests/scripts/run_memorytest.py
View File

@ -17,11 +17,10 @@ import os
import statistics
import sys
import tempfile
import yaml
from pathlib import Path
from pprint import pprint
import yaml
UTILS_DIR = os.path.join(Path(__file__).parent.parent.parent, "utils")
sys.path.insert(0, str(UTILS_DIR))

100
tests/memory_tests/src/memory_tests/memtest_infer.cpp
View File

@ -8,7 +8,6 @@
#include "common_utils.h"
#include "memory_tests_helper/memory_counter.h"
#include "memory_tests_helper/utils.h"
using namespace InferenceEngine;
/**
@ -17,57 +16,56 @@ using namespace InferenceEngine;
* handling it by itself.
*/
int runPipeline(const std::string &model, const std::string &device) {
auto pipeline = [](const std::string &model, const std::string &device) {
Core ie;
CNNNetwork cnnNetwork;
ExecutableNetwork exeNetwork;
InferRequest inferRequest;
size_t batchSize = 0;
auto pipeline = [](const std::string &model, const std::string &device) {
InferenceEngine::Core ie;
InferenceEngine::CNNNetwork cnnNetwork;
InferenceEngine::ExecutableNetwork exeNetwork;
InferenceEngine::InferRequest inferRequest;
size_t batchSize = 0;
ie.GetVersions(device);
MEMORY_SNAPSHOT(load_plugin);
ie.GetVersions(device);
MEMORY_SNAPSHOT(load_plugin);
if (MemoryTest::fileExt(model) == "blob") {
exeNetwork = ie.ImportNetwork(model, device);
MEMORY_SNAPSHOT(import_network);
if (MemoryTest::fileExt(model) == "blob") {
exeNetwork = ie.ImportNetwork(model, device);
MEMORY_SNAPSHOT(import_network);
} else {
cnnNetwork = ie.ReadNetwork(model);
MEMORY_SNAPSHOT(read_network);
exeNetwork = ie.LoadNetwork(cnnNetwork, device);
MEMORY_SNAPSHOT(load_network);
batchSize = cnnNetwork.getBatchSize();
}
MEMORY_SNAPSHOT(create_exenetwork);
inferRequest = exeNetwork.CreateInferRequest();
batchSize = batchSize != 0 ? batchSize : 1;
const InferenceEngine::ConstInputsDataMap inputsInfo(exeNetwork.GetInputsInfo());
fillBlobs(inferRequest, inputsInfo, batchSize);
MEMORY_SNAPSHOT(fill_inputs)
inferRequest.Infer();
MEMORY_SNAPSHOT(first_inference);
MEMORY_SNAPSHOT(full_run);
};
try {
pipeline(model, device);
} catch (const InferenceEngine::Exception &iex) {
std::cerr
<< "Inference Engine pipeline failed with Inference Engine exception:\n"
<< iex.what();
return 1;
} catch (const std::exception &ex) {
std::cerr << "Inference Engine pipeline failed with exception:\n"
<< ex.what();
return 2;
} catch (...) {
std::cerr << "Inference Engine pipeline failed\n";
return 3;
}
else {
cnnNetwork = ie.ReadNetwork(model);
MEMORY_SNAPSHOT(read_network);
exeNetwork = ie.LoadNetwork(cnnNetwork, device);
MEMORY_SNAPSHOT(load_network);
batchSize = cnnNetwork.getBatchSize();
}
MEMORY_SNAPSHOT(create_exenetwork);
inferRequest = exeNetwork.CreateInferRequest();
batchSize = batchSize != 0 ? batchSize : 1;
const InferenceEngine::ConstInputsDataMap inputsInfo(exeNetwork.GetInputsInfo());
fillBlobs(inferRequest, inputsInfo, batchSize);
MEMORY_SNAPSHOT(fill_inputs)
inferRequest.Infer();
MEMORY_SNAPSHOT(first_inference);
MEMORY_SNAPSHOT(full_run);
};
try {
pipeline(model, device);
} catch (const InferenceEngine::Exception &iex) {
std::cerr
<< "Inference Engine pipeline failed with Inference Engine exception:\n"
<< iex.what();
return 1;
} catch (const std::exception &ex) {
std::cerr << "Inference Engine pipeline failed with exception:\n"
<< ex.what();
return 2;
} catch (...) {
std::cerr << "Inference Engine pipeline failed\n";
return 3;
}
return 0;
return 0;
}

71
tests/memory_tests/src/memory_tests/memtest_infer_api_2.cpp Normal file
View File

@ -0,0 +1,71 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <openvino/runtime/infer_request.hpp>
#include <iostream>
#include <fstream>
#include "common_utils.h"
#include "memory_tests_helper/memory_counter.h"
#include "memory_tests_helper/utils.h"
#include "openvino/runtime/core.hpp"
/**
* @brief Function that contain executable pipeline which will be called from
* main(). The function should not throw any exceptions and responsible for
* handling it by itself.
*/
int runPipeline(const std::string &model, const std::string &device) {
auto pipeline = [](const std::string &model, const std::string &device) {
ov::Core ie;
std::shared_ptr<ov::Model> network;
ov::CompiledModel compiled_model;
ov::InferRequest infer_request;
ie.get_versions(device);
MEMORY_SNAPSHOT(load_plugin);
if (MemoryTest::fileExt(model) == "blob") {
std::ifstream streamModel{model};
compiled_model = ie.import_model(streamModel, device);
MEMORY_SNAPSHOT(import_network);
} else {
network = ie.read_model(model);
MEMORY_SNAPSHOT(read_network);
compiled_model = ie.compile_model(network, device);
MEMORY_SNAPSHOT(load_network);
}
MEMORY_SNAPSHOT(create_exenetwork);
infer_request = compiled_model.create_infer_request();
auto inputs = network->inputs();
fillTensors(infer_request, inputs);
MEMORY_SNAPSHOT(fill_inputs)
infer_request.infer();
MEMORY_SNAPSHOT(first_inference);
MEMORY_SNAPSHOT(full_run);
};
try {
pipeline(model, device);
} catch (const InferenceEngine::Exception &iex) {
std::cerr
<< "Inference Engine pipeline failed with Inference Engine exception:\n"
<< iex.what();
return 1;
} catch (const std::exception &ex) {
std::cerr << "Inference Engine pipeline failed with exception:\n"
<< ex.what();
return 2;
} catch (...) {
std::cerr << "Inference Engine pipeline failed\n";
return 3;
}
return 0;
}

38
tests/memory_tests/src/memory_tests_helper/cli.h
View File

@ -14,21 +14,21 @@ static const char help_message[] = "Print a usage message";
/// @brief message for model argument
static const char model_message[] =
"Required. Path to an .xml/.onnx/.prototxt file with a trained model or to "
"a .blob files with a trained compiled model.";
"Required. Path to an .xml/.onnx/.prototxt file with a trained model or to "
"a .blob files with a trained compiled model.";
/// @brief message for target device argument
static const char target_device_message[] =
"Required. Specify a target device to infer on. "
"Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO "
"plugin. "
"Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI "
"plugin. "
"The application looks for a suitable plugin for the specified device.";
"Required. Specify a target device to infer on. "
"Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO "
"plugin. "
"Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI "
"plugin. "
"The application looks for a suitable plugin for the specified device.";
/// @brief message for statistics path argument
static const char statistics_path_message[] =
"Required. Path to a file to write statistics.";
"Required. Path to a file to write statistics.";
/// @brief Define flag for showing help message <br>
DEFINE_bool(h, false, help_message);
@ -52,14 +52,14 @@ DEFINE_string(s, "", statistics_path_message);
* @brief This function show a help message
*/
static void showUsage() {
std::cout << std::endl;
std::cout << "TimeTests [OPTION]" << std::endl;
std::cout << "Options:" << std::endl;
std::cout << std::endl;
std::cout << " -h, --help " << help_message << std::endl;
std::cout << " -m \"<path>\" " << model_message << std::endl;
std::cout << " -d \"<device>\" " << target_device_message
<< std::endl;
std::cout << " -s \"<path>\" " << statistics_path_message
<< std::endl;
std::cout << std::endl;
std::cout << "TimeTests [OPTION]" << std::endl;
std::cout << "Options:" << std::endl;
std::cout << std::endl;
std::cout << " -h, --help " << help_message << std::endl;
std::cout << " -m \"<path>\" " << model_message << std::endl;
std::cout << " -d \"<device>\" " << target_device_message
<< std::endl;
std::cout << " -s \"<path>\" " << statistics_path_message
<< std::endl;
}

42
tests/memory_tests/src/memory_tests_helper/main.cpp
View File

@ -14,25 +14,25 @@ int runPipeline(const std::string &model, const std::string &device);
* @brief Parses command line and check required arguments
*/
bool parseAndCheckCommandLine(int argc, char **argv) {
gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
if (FLAGS_help || FLAGS_h) {
showUsage();
return false;
}
gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
if (FLAGS_help || FLAGS_h) {
showUsage();
return false;
}
if (FLAGS_m.empty())
throw std::logic_error(
"Model is required but not set. Please set -m option.");
if (FLAGS_m.empty())
throw std::logic_error(
"Model is required but not set. Please set -m option.");
if (FLAGS_d.empty())
throw std::logic_error(
"Device is required but not set. Please set -d option.");
if (FLAGS_d.empty())
throw std::logic_error(
"Device is required but not set. Please set -d option.");
if (FLAGS_s.empty())
throw std::logic_error(
"Statistics file path is required but not set. Please set -s option.");
if (FLAGS_s.empty())
throw std::logic_error(
"Statistics file path is required but not set. Please set -s option.");
return true;
return true;
}
/**
@ -48,11 +48,11 @@ int _runPipeline() {
* @brief Main entry point
*/
int main(int argc, char **argv) {
if (!parseAndCheckCommandLine(argc, argv))
return -1;
if (!parseAndCheckCommandLine(argc, argv))
return -1;
auto status = _runPipeline();
StatisticsWriter::Instance().setFile(FLAGS_s);
StatisticsWriter::Instance().write();
return status;
auto status = _runPipeline();
StatisticsWriter::Instance().setFile(FLAGS_s);
StatisticsWriter::Instance().write();
return status;
}

154
tests/memory_tests/src/memory_tests_helper/memory_counter.cpp
View File

@ -14,105 +14,111 @@
#include <psapi.h>
#include <tlhelp32.h>
#else
#include <sys/unistd.h>
#include <sys/wait.h>
#endif
#include "statistics_writer.h"
namespace MemoryTest {
#ifdef _WIN32
static PROCESS_MEMORY_COUNTERS getMemoryInfo() {
static PROCESS_MEMORY_COUNTERS pmc;
pmc.cb = sizeof(PROCESS_MEMORY_COUNTERS);
GetProcessMemoryInfo(GetCurrentProcess(),&pmc, pmc.cb);
return pmc;
}
size_t getVmSizeInKB() {
return getMemoryInfo().PagefileUsage / 1024;
static PROCESS_MEMORY_COUNTERS getMemoryInfo() {
static PROCESS_MEMORY_COUNTERS pmc;
pmc.cb = sizeof(PROCESS_MEMORY_COUNTERS);
GetProcessMemoryInfo(GetCurrentProcess(),&pmc, pmc.cb);
return pmc;
}
size_t getVmPeakInKB() {
return getMemoryInfo().PeakPagefileUsage / 1024;
}
size_t getVmSizeInKB() {
return getMemoryInfo().PagefileUsage / 1024;
}
size_t getVmRSSInKB() {
return getMemoryInfo().WorkingSetSize / 1024;
}
size_t getVmPeakInKB() {
return getMemoryInfo().PeakPagefileUsage / 1024;
}
size_t getVmHWMInKB() {
return getMemoryInfo().PeakWorkingSetSize / 1024;
}
size_t getVmRSSInKB() {
return getMemoryInfo().WorkingSetSize / 1024;
}
size_t getThreadsNum() {
// first determine the id of the current process
DWORD const id = GetCurrentProcessId();
size_t getVmHWMInKB() {
return getMemoryInfo().PeakWorkingSetSize / 1024;
}
// then get a process list snapshot.
HANDLE const snapshot = CreateToolhelp32Snapshot( TH32CS_SNAPALL, 0 );
size_t getThreadsNum() {
// first determine the id of the current process
DWORD const id = GetCurrentProcessId();
// initialize the process entry structure.
PROCESSENTRY32 entry = { 0 };
entry.dwSize = sizeof( entry );
// then get a process list snapshot.
HANDLE const snapshot = CreateToolhelp32Snapshot( TH32CS_SNAPALL, 0 );
// get the first process info.
BOOL ret = true;
ret = Process32First( snapshot, &entry );
while( ret && entry.th32ProcessID != id ) {
ret = Process32Next( snapshot, &entry );
}
CloseHandle( snapshot );
return ret
? entry.cntThreads
: -1;
}
// initialize the process entry structure.
PROCESSENTRY32 entry = { 0 };
entry.dwSize = sizeof( entry );
// get the first process info.
BOOL ret = true;
ret = Process32First( snapshot, &entry );
while( ret && entry.th32ProcessID != id ) {
ret = Process32Next( snapshot, &entry );
}
CloseHandle( snapshot );
return ret
? entry.cntThreads
: -1;
}
#else
/// Parses number from provided string
static int parseLine(std::string line) {
std::string res = "";
for (auto c: line)
if (isdigit(c))
res += c;
if (res.empty())
// If number wasn't found return -1
return -1;
return std::stoi(res);
}
size_t getSystemDataByName(char *name){
FILE* file = fopen("/proc/self/status", "r");
size_t result = 0;
if (file != nullptr) {
char line[128];
while (fgets(line, 128, file) != NULL) {
if (strncmp(line, name, strlen(name)) == 0) {
result = parseLine(line);
break;
}
}
fclose(file);
static int parseLine(std::string line) {
std::string res = "";
for (auto c: line)
if (isdigit(c))
res += c;
if (res.empty())
// If number wasn't found return -1
return -1;
return std::stoi(res);
}
return result;
}
size_t getVmSizeInKB() {return getSystemDataByName((char*) "VmSize:");}
size_t getVmPeakInKB() {return getSystemDataByName((char*) "VmPeak:");}
size_t getVmRSSInKB() {return getSystemDataByName((char*) "VmRSS:");}
size_t getVmHWMInKB() {return getSystemDataByName((char*) "VmHWM:");}
size_t getThreadsNum() {return getSystemDataByName((char*) "Threads:");}
size_t getSystemDataByName(char *name) {
FILE *file = fopen("/proc/self/status", "r");
size_t result = 0;
if (file != nullptr) {
char line[128];
while (fgets(line, 128, file) != NULL) {
if (strncmp(line, name, strlen(name)) == 0) {
result = parseLine(line);
break;
}
}
fclose(file);
}
return result;
}
size_t getVmSizeInKB() { return getSystemDataByName((char *) "VmSize:"); }
size_t getVmPeakInKB() { return getSystemDataByName((char *) "VmPeak:"); }
size_t getVmRSSInKB() { return getSystemDataByName((char *) "VmRSS:"); }
size_t getVmHWMInKB() { return getSystemDataByName((char *) "VmHWM:"); }
size_t getThreadsNum() { return getSystemDataByName((char *) "Threads:"); }
#endif
MemoryCounter::MemoryCounter(const std::string &mem_counter_name) {
name = mem_counter_name;
std::vector<size_t> memory_measurements = {getVmRSSInKB(), getVmHWMInKB(), getVmSizeInKB(),
getVmPeakInKB(), getThreadsNum()};
StatisticsWriter::Instance().addMemCounterToStructure({name, memory_measurements});
}
MemoryCounter::MemoryCounter(const std::string &mem_counter_name) {
name = mem_counter_name;
std::vector<size_t> memory_measurements = {getVmRSSInKB(), getVmHWMInKB(), getVmSizeInKB(),
getVmPeakInKB(), getThreadsNum()};
StatisticsWriter::Instance().addMemCounterToStructure({name, memory_measurements});
}
} // namespace MemoryTest

98
tests/memory_tests/src/memory_tests_helper/statistics_writer.h
View File

@ -22,59 +22,61 @@
*/
class StatisticsWriter {
private:
std::ofstream statistics_file;
std::ofstream statistics_file;
std::map<std::string, std::vector<size_t>> mem_structure; // mem_counter_name, memory measurements
std::vector<std::string> mem_struct_order;
std::map<std::string, std::vector<size_t>> mem_structure; // mem_counter_name, memory measurements
std::vector<std::string> mem_struct_order;
StatisticsWriter() = default;
StatisticsWriter(const StatisticsWriter &) = delete;
StatisticsWriter &operator=(const StatisticsWriter &) = delete;
StatisticsWriter() = default;
StatisticsWriter(const StatisticsWriter &) = delete;
StatisticsWriter &operator=(const StatisticsWriter &) = delete;
public:
/**
* @brief Creates StatisticsWriter singleton object
*/
static StatisticsWriter &Instance() {
static StatisticsWriter writer;
return writer;
}
/**
* @brief Specifies, opens and validates statistics path for writing
*/
void setFile(const std::string &statistics_path) {
statistics_file.open(statistics_path);
if (!statistics_file.good()) {
std::stringstream err;
err << "Statistic file \"" << statistics_path
<< "\" can't be used for writing";
throw std::runtime_error(err.str());
/**
* @brief Creates StatisticsWriter singleton object
*/
static StatisticsWriter &Instance() {
static StatisticsWriter writer;
return writer;
}
}
/**
* @brief Creates counter structure
*/
void addMemCounterToStructure(const std::pair<std::string, std::vector<size_t>> &record) {
mem_struct_order.push_back(record.first);
mem_structure[record.first] = record.second;
}
/**
* @brief Writes provided statistics in YAML format.
*/
void write() {
if (!statistics_file)
throw std::runtime_error("Statistic file path isn't set");
for (auto& mem_counter: mem_struct_order) {
statistics_file << mem_counter << ":" << '\n'
<< SEPARATOR << "vmrss: " << mem_structure[mem_counter][0] << '\n'
<< SEPARATOR << "vmhwm: " << mem_structure[mem_counter][1] << '\n'
<< SEPARATOR << "vmsize: " << mem_structure[mem_counter][2] << '\n'
<< SEPARATOR << "vmpeak: " << mem_structure[mem_counter][3] << '\n'
<< SEPARATOR << "threads: " << mem_structure[mem_counter][4] << '\n';
/**
* @brief Specifies, opens and validates statistics path for writing
*/
void setFile(const std::string &statistics_path) {
statistics_file.open(statistics_path);
if (!statistics_file.good()) {
std::stringstream err;
err << "Statistic file \"" << statistics_path
<< "\" can't be used for writing";
throw std::runtime_error(err.str());
}
}
/**
* @brief Creates counter structure
*/
void addMemCounterToStructure(const std::pair<std::string, std::vector<size_t>> &record) {
mem_struct_order.push_back(record.first);
mem_structure[record.first] = record.second;
}
/**
* @brief Writes provided statistics in YAML format.
*/
void write() {
if (!statistics_file)
throw std::runtime_error("Statistic file path isn't set");
for (auto &mem_counter: mem_struct_order) {
statistics_file << mem_counter << ":" << '\n'
<< SEPARATOR << "vmrss: " << mem_structure[mem_counter][0] << '\n'
<< SEPARATOR << "vmhwm: " << mem_structure[mem_counter][1] << '\n'
<< SEPARATOR << "vmsize: " << mem_structure[mem_counter][2] << '\n'
<< SEPARATOR << "vmpeak: " << mem_structure[mem_counter][3] << '\n'
<< SEPARATOR << "threads: " << mem_structure[mem_counter][4] << '\n';
}
statistics_file << "---" << '\n' << "measurement_unit: Kb";
}
statistics_file << "---" << '\n' << "measurement_unit: Kb";
}
};

12
tests/memory_tests/test_runner/conftest.py
View File

@ -18,17 +18,16 @@ This plugin adds the following command-line options:
import hashlib
import json
import logging
import tempfile
# pylint:disable=import-error
import os
import pytest
import sys
import tempfile
import yaml
from copy import deepcopy
from inspect import getsourcefile
from pathlib import Path
import pytest
import yaml
from jsonschema import validate, ValidationError
from pathlib import Path
UTILS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "utils")
sys.path.insert(0, str(UTILS_DIR))
@ -43,7 +42,6 @@ sys.path.append(MEMORY_TESTS_DIR)
from test_runner.utils import query_memory_timeline, REFS_FACTOR
OMZ_NUM_ATTEMPTS = 6
@ -168,6 +166,8 @@ def executable(request):
def niter(request):
"""Fixture function for command-line option."""
return request.config.getoption('niter')
# -------------------- CLI options --------------------

5
tests/memory_tests/test_runner/utils.py
View File

@ -37,6 +37,7 @@ def _transpose_dicts(items, template=None):
def query_memory_timeline(records, db_url, db_name, db_collection, max_items=20, similarity=TIMELINE_SIMILARITY):
""" Query database for similar memory items committed previously
"""
def timeline_key(item):
""" Defines order for timeline report entries
"""
@ -45,10 +46,10 @@ def query_memory_timeline(records, db_url, db_name, db_collection, max_items=20,
if len(item['results'][step_name]['vmhwm']) <= 1:
return 1
order = item['results'][step_name]['vmhwm']["avg"][-1] - item['results'][step_name]['vmhwm']["avg"][-2] + \
item['results'][step_name]['vmrss']["avg"][-1] - item['results'][step_name]['vmrss']["avg"][-2]
item['results'][step_name]['vmrss']["avg"][-1] - item['results'][step_name]['vmrss']["avg"][-2]
if not item['status']:
# ensure failed cases are always on top
order += sys.maxsize/2
order += sys.maxsize / 2
return order
client = MongoClient(db_url)