Add benchmark samples (#13388)
* Add sync_bnehcmark * Fix Unix comilation * niter->time * Explain main loop * samples: factor out common * Code style * clang-format -i * return 0; -> return EXIT_SUCCESS;, +x * Update throughput_benchmark * Add READMEs * Fix READMEs refs * Add sync_benchmark.py * Add niter, infer_new_request, -pc * from datetime import timedelta * Fix niter and seconds_to_run * Add disclaimer about benchmark_app performance * Update samples/cpp/benchmark/sync_benchmark/README.md * Add dynamic_shape_bert_benhcmark * Add dynamic_shape_detection_benchmark * Adopt for detr-resnet50 * Remove sync_benchmark2, throughput_benchmark2, perf counters * clang-format -i * Fix flake8 * Add README.md * Add links to sample_dynamic_shape_bert_benchmark * Add softmax * nameless LatencyMetrics * parent.parent -> parents[2] * Add bert_benhcmark sample * Code style * Add bert_benhcmark/README.md * rm -r samples/python/benchmark/dynamic_shape_bert_benhcmark/ * rm -r samples/cpp/benchmark/dynamic_shape_detection_benchmark/ * bert_benhcmark/README.md: remove dynamic shape * Remove add_subdirectory(dynamic_shape_detection_benchmark) * flake8 * samples: Add a note about CUMULATIVE_THROUGHPUT, don’t expect get_property() to throw, don’t introduce json dependency for samples/cpp/common * / namespace * Add article * namespace -> static * Update README, seconds_ro_run 10, niter 10, no inter alinment * percentile->median * benchmark samples: use generate(), align logs, update READMEs * benchmakr samples: remove percentile() * samples/python/benchmark/bert_benhcmark/bert_benhcmark.py: report average sequence length and processing time * Python samples: move requirements.txt to every sample * Remove numpy from requirements.txt * Remove Building section from Python samples, install only required extras from openvino-dev, set up environment for bert_benhcmark, report duration for bert_benhcmark * Install openvino-dev for Hello Reshape SSD C++ Sample
This commit is contained in:
Zlobin Vladimir authored and 
GitHub committed
1 parent
a96f2af2f4
commit
67d25d7099
33 files changed
+1102
-103
No files matched your search
@@ -0,0 +1,50 @@
|
||||
# Bert Benchmark Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_bert_benchmark_README}
|
||||
|
||||
This sample demonstrates how to estimate performace of a Bert model using Asynchronous Inference Request API. Unlike [demos](@ref omz_demos) this sample doesn't have configurable command line arguments. Feel free to modify sample's source code to try out different options.
|
||||
|
||||
The following Python\* API is used in the application:
|
||||
|
||||
| Feature | API | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version |
|
||||
| Basic Infer Flow | [openvino.runtime.Core], [openvino.runtime.Core.compile_model] | Common API to do inference: compile a model |
|
||||
| Asynchronous Infer | [openvino.runtime.AsyncInferQueue], [openvino.runtime.AsyncInferQueue.start_async], [openvino.runtime.AsyncInferQueue.wait_all] | Do asynchronous inference |
|
||||
| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model |
|
||||
|
||||
## How It Works
|
||||
|
||||
The sample downloads a model and a tokenizer, export the model to onnx, reads the exported model and reshapes it to enforce dynamic inpus shapes, compiles the resulting model, downloads a dataset and runs benhcmarking on the dataset.
|
||||
|
||||
You can see the explicit description of
|
||||
each sample step at [Integration Steps](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide.
|
||||
|
||||
## Running
|
||||
|
||||
Install the `openvino` Python package:
|
||||
|
||||
```
|
||||
python -m pip install openvino
|
||||
```
|
||||
|
||||
Install packages from `requirements.txt`:
|
||||
|
||||
```
|
||||
python -m pip install -r requirements.txt
|
||||
```
|
||||
|
||||
Run the sample
|
||||
|
||||
```
|
||||
python bert_benhcmark.py
|
||||
```
|
||||
|
||||
## Sample Output
|
||||
|
||||
The sample outputs how long it takes to process a dataset.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Integrate the OpenVINO™ Runtime with Your Application](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md)
|
||||
- [Using OpenVINO™ Toolkit Samples](../../../../docs/OV_Runtime_UG/Samples_Overview.md)
|
||||
- [Model Downloader](@ref omz_tools_downloader)
|
||||
- [Model Optimizer](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md)
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (C) 2022 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging as log
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from time import perf_counter
|
||||
|
||||
import datasets
|
||||
from openvino.runtime import Core, get_version, AsyncInferQueue, PartialShape
|
||||
from transformers import AutoTokenizer
|
||||
from transformers.onnx import export
|
||||
from transformers.onnx.features import FeaturesManager
|
||||
|
||||
|
||||
def main():
|
||||
log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout)
|
||||
log.info('OpenVINO:')
|
||||
log.info(f"{'Build ':.<39} {get_version()}")
|
||||
model_name = 'bert-base-uncased'
|
||||
# Download the model
|
||||
transformers_model = FeaturesManager.get_model_from_feature('default', model_name)
|
||||
_, model_onnx_config = FeaturesManager.check_supported_model_or_raise(transformers_model, feature='default')
|
||||
onnx_config = model_onnx_config(transformers_model.config)
|
||||
# Download the tokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
|
||||
core = Core()
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
onnx_path = Path(tmp) / f'{model_name}.onnx'
|
||||
# Export .onnx
|
||||
export(tokenizer, transformers_model, onnx_config, onnx_config.default_onnx_opset, onnx_path)
|
||||
# Read .onnx with OpenVINO
|
||||
model = core.read_model(onnx_path)
|
||||
|
||||
# Enforce dynamic input shape
|
||||
try:
|
||||
model.reshape({model_input.any_name: PartialShape([1, '?']) for model_input in model.inputs})
|
||||
except RuntimeError:
|
||||
log.error("Can't set dynamic shape")
|
||||
raise
|
||||
# Optimize for throughput. Best throughput can be reached by
|
||||
# running multiple openvino.runtime.InferRequest instances asyncronously
|
||||
tput = {'PERFORMANCE_HINT': 'THROUGHPUT'}
|
||||
# Pick a device by replacing CPU, for example MULTI:CPU(4),GPU(8).
|
||||
# It is possible to set CUMULATIVE_THROUGHPUT as PERFORMANCE_HINT for AUTO device
|
||||
compiled_model = core.compile_model(model, 'CPU', tput)
|
||||
# AsyncInferQueue creates optimal number of InferRequest instances
|
||||
ireqs = AsyncInferQueue(compiled_model)
|
||||
|
||||
sst2 = datasets.load_dataset('glue', 'sst2')
|
||||
sst2_sentences = sst2['validation']['sentence']
|
||||
# Warm up
|
||||
encoded_warm_up = dict(tokenizer('Warm up sentence is here.', return_tensors='np'))
|
||||
for _ in ireqs:
|
||||
ireqs.start_async(encoded_warm_up)
|
||||
ireqs.wait_all()
|
||||
# Benchmark
|
||||
sum_seq_len = 0
|
||||
start = perf_counter()
|
||||
for sentence in sst2_sentences:
|
||||
encoded = dict(tokenizer(sentence, return_tensors='np'))
|
||||
sum_seq_len += next(iter(encoded.values())).size # get sequence length to compute average length
|
||||
ireqs.start_async(encoded)
|
||||
ireqs.wait_all()
|
||||
end = perf_counter()
|
||||
duration = end - start
|
||||
log.info(f'Average sequence length: {sum_seq_len / len(sst2_sentences):.2f}')
|
||||
log.info(f'Average processing time: {duration / len(sst2_sentences) * 1e3:.2f} ms')
|
||||
log.info(f'Duration: {duration:.2f} seconds')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,2 @@
|
||||
transformers[onnx]
|
||||
torch
|
||||
@@ -0,0 +1,92 @@
|
||||
# Sync Benchmark Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_sync_benchmark_README}
|
||||
|
||||
This sample demonstrates how to estimate performace of a model using Synchronous Inference Request API. It makes sence to use synchronous inference only in latency oriented scenarios. Models with static input shapes are supported. Unlike [demos](@ref omz_demos) this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options.
|
||||
|
||||
The following Python\* API is used in the application:
|
||||
|
||||
| Feature | API | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version |
|
||||
| Basic Infer Flow | [openvino.runtime.Core], [openvino.runtime.Core.compile_mode], [openvino.runtime.InferRequest.get_tensor] | Common API to do inference: compile a model, configure input tensors |
|
||||
| Synchronous Infer | [openvino.runtime.InferRequest.infer] | Do synchronous inference |
|
||||
| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model |
|
||||
| Tensor Operations | [openvino.runtime.Tensor.get_shape], [openvino.runtime.Tensor.data] | Get a tensor shape and its data. |
|
||||
|
||||
| Options | Values |
|
||||
| :--- | :--- |
|
||||
| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) [yolo-v3-tf](@ref omz_models_model_yolo_v3_tf), [face-detection-0200](@ref omz_models_model_face_detection_0200) |
|
||||
| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) |
|
||||
| Supported devices | [All](../../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) |
|
||||
| Other language realization | [C++](../../../cpp/benchmark/sync_benchmark/README.md) |
|
||||
|
||||
## How It Works
|
||||
|
||||
The sample compiles a model for a given device, randomly generates input data, performs synchronous inference multiple times for a given number of seconds. Then processes and reports performance results.
|
||||
|
||||
You can see the explicit description of
|
||||
each sample step at [Integration Steps](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide.
|
||||
|
||||
## Running
|
||||
|
||||
```
|
||||
python sync_benchmark.py <path_to_model>
|
||||
```
|
||||
|
||||
To run the sample, you need to specify a model:
|
||||
- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader).
|
||||
|
||||
> **NOTES**:
|
||||
>
|
||||
> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md).
|
||||
>
|
||||
> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing.
|
||||
|
||||
### Example
|
||||
|
||||
1. Install the `openvino-dev` Python package to use Open Model Zoo Tools:
|
||||
|
||||
```
|
||||
python -m pip install openvino-dev[caffe]
|
||||
```
|
||||
|
||||
2. Download a pre-trained model using:
|
||||
|
||||
```
|
||||
omz_downloader --name googlenet-v1
|
||||
```
|
||||
|
||||
3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter:
|
||||
|
||||
```
|
||||
omz_converter --name googlenet-v1
|
||||
```
|
||||
|
||||
4. Perform benchmarking using the `googlenet-v1` model on a `CPU`:
|
||||
|
||||
```
|
||||
python sync_benchmark.py googlenet-v1.xml
|
||||
```
|
||||
|
||||
## Sample Output
|
||||
|
||||
The application outputs performance results.
|
||||
|
||||
```
|
||||
[ INFO ] OpenVINO:
|
||||
[ INFO ] Build ................................. <version>
|
||||
[ INFO ] Count: 2333 iterations
|
||||
[ INFO ] Duration: 10003.59 ms
|
||||
[ INFO ] Latency:
|
||||
[ INFO ] Median: 3.90 ms
|
||||
[ INFO ] Average: 4.29 ms
|
||||
[ INFO ] Min: 3.30 ms
|
||||
[ INFO ] Max: 10.11 ms
|
||||
[ INFO ] Throughput: 233.22 FPS
|
||||
```
|
||||
|
||||
## See Also
|
||||
|
||||
- [Integrate the OpenVINO™ Runtime with Your Application](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md)
|
||||
- [Using OpenVINO™ Toolkit Samples](../../../../docs/OV_Runtime_UG/Samples_Overview.md)
|
||||
- [Model Downloader](@ref omz_tools_downloader)
|
||||
- [Model Optimizer](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md)
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (C) 2022 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging as log
|
||||
import statistics
|
||||
import sys
|
||||
from time import perf_counter
|
||||
|
||||
import numpy as np
|
||||
from openvino.runtime import Core, get_version
|
||||
from openvino.runtime.utils.types import get_dtype
|
||||
|
||||
|
||||
def fill_tensor_random(tensor):
|
||||
dtype = get_dtype(tensor.element_type)
|
||||
rand_min, rand_max = (0, 1) if dtype == bool else (np.iinfo(np.uint8).min, np.iinfo(np.uint8).max)
|
||||
# np.random.uniform excludes high: add 1 to have it generated
|
||||
if np.dtype(dtype).kind in ['i', 'u', 'b']:
|
||||
rand_max += 1
|
||||
rs = np.random.RandomState(np.random.MT19937(np.random.SeedSequence(0)))
|
||||
if 0 == tensor.get_size():
|
||||
raise RuntimeError("Models with dynamic shapes aren't supported. Input tensors must have specific shapes before inference")
|
||||
tensor.data[:] = rs.uniform(rand_min, rand_max, list(tensor.shape)).astype(dtype)
|
||||
|
||||
|
||||
def main():
|
||||
log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout)
|
||||
log.info('OpenVINO:')
|
||||
log.info(f"{'Build ':.<39} {get_version()}")
|
||||
if len(sys.argv) != 2:
|
||||
log.info(f'Usage: {sys.argv[0]} <path_to_model>')
|
||||
return 1
|
||||
# Optimize for latency. Most of the devices are configured for latency by default,
|
||||
# but there are exceptions like MYRIAD
|
||||
latency = {'PERFORMANCE_HINT': 'LATENCY'}
|
||||
|
||||
# Create Core and use it to compile a model.
|
||||
# Pick a device by replacing CPU, for example AUTO:GPU,CPU.
|
||||
# Using MULTI device is pointless in sync scenario
|
||||
# because only one instance of openvino.runtime.InferRequest is used
|
||||
core = Core()
|
||||
compiled_model = core.compile_model(sys.argv[1], 'CPU', latency)
|
||||
ireq = compiled_model.create_infer_request()
|
||||
# Fill input data for the ireq
|
||||
for model_input in compiled_model.inputs:
|
||||
fill_tensor_random(ireq.get_tensor(model_input))
|
||||
# Warm up
|
||||
ireq.infer()
|
||||
# Benchmark for seconds_to_run seconds and at least niter iterations
|
||||
seconds_to_run = 10
|
||||
niter = 10
|
||||
latencies = []
|
||||
start = perf_counter()
|
||||
time_point = start
|
||||
time_point_to_finish = start + seconds_to_run
|
||||
while time_point < time_point_to_finish or len(latencies) < niter:
|
||||
ireq.infer()
|
||||
iter_end = perf_counter()
|
||||
latencies.append((iter_end - time_point) * 1e3)
|
||||
time_point = iter_end
|
||||
end = time_point
|
||||
duration = end - start
|
||||
# Report results
|
||||
fps = len(latencies) / duration
|
||||
log.info(f'Count: {len(latencies)} iterations')
|
||||
log.info(f'Duration: {duration * 1e3:.2f} ms')
|
||||
log.info('Latency:')
|
||||
log.info(f' Median: {statistics.median(latencies):.2f} ms')
|
||||
log.info(f' Average: {sum(latencies) / len(latencies):.2f} ms')
|
||||
log.info(f' Min: {min(latencies):.2f} ms')
|
||||
log.info(f' Max: {max(latencies):.2f} ms')
|
||||
log.info(f'Throughput: {fps:.2f} FPS')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
# Throughput Benchmark Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_throughput_benchmark_README}
|
||||
|
||||
This sample demonstrates how to estimate performace of a model using Asynchronous Inference Request API in throughput mode. Unlike [demos](@ref omz_demos) this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options.
|
||||
|
||||
The reported results may deviate from what [benchmark_app](../../../../tools/benchmark_tool/README.md) reports. One example is model input precision for computer vision tasks. benchmark_app sets uint8, while the sample uses default model precision which is usually float32.
|
||||
|
||||
The following Python\* API is used in the application:
|
||||
|
||||
| Feature | API | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version |
|
||||
| Basic Infer Flow | [openvino.runtime.Core], [openvino.runtime.Core.compile_model], [openvino.runtime.InferRequest.get_tensor] | Common API to do inference: compile a model, configure input tensors |
|
||||
| Asynchronous Infer | [openvino.runtime.AsyncInferQueue], [openvino.runtime.AsyncInferQueue.start_async], [openvino.runtime.AsyncInferQueue.wait_all], [openvino.runtime.InferRequest.results] | Do asynchronous inference |
|
||||
| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model |
|
||||
| Tensor Operations | [openvino.runtime.Tensor.get_shape], [openvino.runtime.Tensor.data] | Get a tensor shape and its data. |
|
||||
|
||||
| Options | Values |
|
||||
| :--- | :--- |
|
||||
| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) [yolo-v3-tf](@ref omz_models_model_yolo_v3_tf), [face-detection-0200](@ref omz_models_model_face_detection_0200) |
|
||||
| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) |
|
||||
| Supported devices | [All](../../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) |
|
||||
| Other language realization | [C++](../../../cpp/benchmark/throughput_benchmark/README.md) |
|
||||
|
||||
## How It Works
|
||||
|
||||
The sample compiles a model for a given device, randomly generates input data, performs asynchronous inference multiple times for a given number of seconds. Then processes and reports performance results.
|
||||
|
||||
You can see the explicit description of
|
||||
each sample step at [Integration Steps](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide.
|
||||
|
||||
## Running
|
||||
|
||||
```
|
||||
python throughput_benchmark.py <path_to_model>
|
||||
```
|
||||
|
||||
To run the sample, you need to specify a model:
|
||||
- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader).
|
||||
|
||||
> **NOTES**:
|
||||
>
|
||||
> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md).
|
||||
>
|
||||
> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing.
|
||||
|
||||
### Example
|
||||
|
||||
1. Install the `openvino-dev` Python package to use Open Model Zoo Tools:
|
||||
|
||||
```
|
||||
python -m pip install openvino-dev[caffe]
|
||||
```
|
||||
|
||||
2. Download a pre-trained model using:
|
||||
|
||||
```
|
||||
omz_downloader --name googlenet-v1
|
||||
```
|
||||
|
||||
3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter:
|
||||
|
||||
```
|
||||
omz_converter --name googlenet-v1
|
||||
```
|
||||
|
||||
4. Perform benchmarking using the `googlenet-v1` model on a `CPU`:
|
||||
|
||||
```
|
||||
python throughput_benchmark.py googlenet-v1.xml
|
||||
```
|
||||
|
||||
## Sample Output
|
||||
|
||||
The application outputs performance results.
|
||||
|
||||
```
|
||||
[ INFO ] OpenVINO:
|
||||
[ INFO ] Build ................................. <version>
|
||||
[ INFO ] Count: 2817 iterations
|
||||
[ INFO ] Duration: 10012.65 ms
|
||||
[ INFO ] Latency:
|
||||
[ INFO ] Median: 13.80 ms
|
||||
[ INFO ] Average: 14.10 ms
|
||||
[ INFO ] Min: 8.35 ms
|
||||
[ INFO ] Max: 28.38 ms
|
||||
[ INFO ] Throughput: 281.34 FPS
|
||||
```
|
||||
|
||||
## See Also
|
||||
|
||||
- [Integrate the OpenVINO™ Runtime with Your Application](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md)
|
||||
- [Using OpenVINO™ Toolkit Samples](../../../../docs/OV_Runtime_UG/Samples_Overview.md)
|
||||
- [Model Downloader](@ref omz_tools_downloader)
|
||||
- [Model Optimizer](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md)
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (C) 2022 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging as log
|
||||
import sys
|
||||
import statistics
|
||||
from time import perf_counter
|
||||
|
||||
import numpy as np
|
||||
from openvino.runtime import Core, get_version, AsyncInferQueue
|
||||
from openvino.runtime.utils.types import get_dtype
|
||||
|
||||
|
||||
def fill_tensor_random(tensor):
|
||||
dtype = get_dtype(tensor.element_type)
|
||||
rand_min, rand_max = (0, 1) if dtype == bool else (np.iinfo(np.uint8).min, np.iinfo(np.uint8).max)
|
||||
# np.random.uniform excludes high: add 1 to have it generated
|
||||
if np.dtype(dtype).kind in ['i', 'u', 'b']:
|
||||
rand_max += 1
|
||||
rs = np.random.RandomState(np.random.MT19937(np.random.SeedSequence(0)))
|
||||
if 0 == tensor.get_size():
|
||||
raise RuntimeError("Models with dynamic shapes aren't supported. Input tensors must have specific shapes before inference")
|
||||
tensor.data[:] = rs.uniform(rand_min, rand_max, list(tensor.shape)).astype(dtype)
|
||||
|
||||
|
||||
def main():
|
||||
log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout)
|
||||
log.info('OpenVINO:')
|
||||
log.info(f"{'Build ':.<39} {get_version()}")
|
||||
if len(sys.argv) != 2:
|
||||
log.info(f'Usage: {sys.argv[0]} <path_to_model>')
|
||||
return 1
|
||||
# Optimize for throughput. Best throughput can be reached by
|
||||
# running multiple openvino.runtime.InferRequest instances asyncronously
|
||||
tput = {'PERFORMANCE_HINT': 'THROUGHPUT'}
|
||||
|
||||
# Create Core and use it to compile a model.
|
||||
# Pick a device by replacing CPU, for example MULTI:CPU(4),GPU(8).
|
||||
# It is possible to set CUMULATIVE_THROUGHPUT as PERFORMANCE_HINT for AUTO device
|
||||
core = Core()
|
||||
compiled_model = core.compile_model(sys.argv[1], 'CPU', tput)
|
||||
# AsyncInferQueue creates optimal number of InferRequest instances
|
||||
ireqs = AsyncInferQueue(compiled_model)
|
||||
# Fill input data for ireqs
|
||||
for ireq in ireqs:
|
||||
for model_input in compiled_model.inputs:
|
||||
fill_tensor_random(ireq.get_tensor(model_input))
|
||||
# Warm up
|
||||
for _ in ireqs:
|
||||
ireqs.start_async()
|
||||
ireqs.wait_all()
|
||||
# Benchmark for seconds_to_run seconds and at least niter iterations
|
||||
seconds_to_run = 10
|
||||
niter = 10
|
||||
latencies = []
|
||||
in_fly = set()
|
||||
start = perf_counter()
|
||||
time_point_to_finish = start + seconds_to_run
|
||||
while perf_counter() < time_point_to_finish or len(latencies) + len(in_fly) < niter:
|
||||
idle_id = ireqs.get_idle_request_id()
|
||||
if idle_id in in_fly:
|
||||
latencies.append(ireqs[idle_id].latency)
|
||||
else:
|
||||
in_fly.add(idle_id)
|
||||
ireqs.start_async()
|
||||
ireqs.wait_all()
|
||||
duration = perf_counter() - start
|
||||
for infer_request_id in in_fly:
|
||||
latencies.append(ireqs[infer_request_id].latency)
|
||||
# Report results
|
||||
fps = len(latencies) / duration
|
||||
log.info(f'Count: {len(latencies)} iterations')
|
||||
log.info(f'Duration: {duration * 1e3:.2f} ms')
|
||||
log.info('Latency:')
|
||||
log.info(f' Median: {statistics.median(latencies):.2f} ms')
|
||||
log.info(f' Average: {sum(latencies) / len(latencies):.2f} ms')
|
||||
log.info(f' Min: {min(latencies):.2f} ms')
|
||||
log.info(f' Max: {max(latencies):.2f} ms')
|
||||
log.info(f'Throughput: {fps:.2f} FPS')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user