diff --git a/tests/lib/src/common_utils.cpp b/tests/lib/src/common_utils.cpp index 3f445a2826f..81b0e828ca0 100644 --- a/tests/lib/src/common_utils.cpp +++ b/tests/lib/src/common_utils.cpp @@ -4,9 +4,6 @@ #include "common_utils.h" -#include -#include - /** * @brief Fill InferRequest blobs with random values or image information @@ -15,6 +12,7 @@ void fillBlobs(InferenceEngine::InferRequest inferRequest, const InferenceEngine::ConstInputsDataMap &inputsInfo, const size_t &batchSize) { std::vector> input_image_sizes; + for (const InferenceEngine::ConstInputsDataMap::value_type &item: inputsInfo) { if (isImage(item.second)) input_image_sizes.push_back(getTensorHeightWidth(item.second->getTensorDesc())); @@ -22,6 +20,7 @@ void fillBlobs(InferenceEngine::InferRequest inferRequest, for (const InferenceEngine::ConstInputsDataMap::value_type &item: inputsInfo) { InferenceEngine::Blob::Ptr inputBlob = inferRequest.GetBlob(item.first); + if (isImageInfo(inputBlob) && (input_image_sizes.size() == 1)) { // Fill image information auto image_size = input_image_sizes.at(0); @@ -57,32 +56,4 @@ void fillBlobs(InferenceEngine::InferRequest inferRequest, throw std::logic_error("Input precision is not supported for " + item.first); } } -} - -/** - * @brief Fill infer_request tensors with random values or image information - */ -void fillTensors(ov::InferRequest &infer_request, const std::vector> &inputs) { - for (size_t i = 0; i < inputs.size(); ++i) { - ov::Tensor input_tensor; - if (inputs[i].get_element_type() == ov::element::f32) { - input_tensor = fillTensorRandom(inputs[i]); - } else if (inputs[i].get_element_type() == ov::element::f16) { - input_tensor = fillTensorRandom(inputs[i]); - } else if (inputs[i].get_element_type() == ov::element::i32) { - input_tensor = fillTensorRandom(inputs[i]); - } else if (inputs[i].get_element_type() == ov::element::u8) { - input_tensor = fillTensorRandom(inputs[i]); - } else if (inputs[i].get_element_type() == ov::element::i8) { - input_tensor = fillTensorRandom(inputs[i]); - } else if (inputs[i].get_element_type() == ov::element::u16) { - input_tensor = fillTensorRandom(inputs[i]); - } else if (inputs[i].get_element_type() == ov::element::i16) { - input_tensor = fillTensorRandom(inputs[i]); - } else { - throw std::logic_error( - "Input precision is not supported for " + inputs[i].get_element_type().get_type_name()); - } - infer_request.set_input_tensor(i, input_tensor); - } -} +} \ No newline at end of file diff --git a/tests/lib/src/common_utils.h b/tests/lib/src/common_utils.h index f05d0a12413..456e895d6f0 100644 --- a/tests/lib/src/common_utils.h +++ b/tests/lib/src/common_utils.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include @@ -67,7 +68,7 @@ inline std::pair getTensorHeightWidth(const InferenceEngine::Ten */ template void fillBlobRandom(InferenceEngine::Blob::Ptr &inputBlob) { - auto minput = InferenceEngine::as(inputBlob); + InferenceEngine::MemoryBlob::Ptr minput = InferenceEngine::as(inputBlob); // locked memory holder should be alive all time while access to its buffer happens auto minputHolder = minput->wmap(); @@ -78,23 +79,26 @@ void fillBlobRandom(InferenceEngine::Blob::Ptr &inputBlob) { } } + /** * @brief Fill InferenceEngine tensor with random values (OV API 2.0) */ -template -ov::Tensor fillTensorRandom(const ov::Output &input) { +template +ov::Tensor fillTensorRandom(T &input) { ov::Tensor tensor{input.get_element_type(), input.get_shape()}; - std::vector values(ov::shape_size(input.get_shape())); - for (size_t i = 0; i < values.size(); ++i) { - values[i] = 1 + static_cast (rand()) / (static_cast (RAND_MAX / (std::numeric_limits::max() - 1))); - } - std::memcpy(tensor.data(), values.data(), sizeof(T) * values.size()); + std::vector values(ov::shape_size(input.get_shape())); + + for (size_t i = 0; i < values.size(); ++i) + values[i] = 1 + static_cast (rand()) / (static_cast (RAND_MAX / (std::numeric_limits::max() - 1))); + + std::memcpy(tensor.data(), values.data(), sizeof(U) * values.size()); + return tensor; } /** - * @brief Fill InferenceEngine blob with image information (OV API 1.0) + * @brief Fill InferenceEngine blob with image information (OV API 1) */ template void fillBlobImInfo(InferenceEngine::Blob::Ptr &inputBlob, @@ -121,14 +125,39 @@ void fillBlobImInfo(InferenceEngine::Blob::Ptr &inputBlob, /** - * @brief Fill InferRequest blobs with random values or image information + * @brief Fill infer_request tensors with random values (OV API 2) + */ +template +void fillTensors(ov::InferRequest &infer_request, std::vector &inputs) { + for (size_t i = 0; i < inputs.size(); ++i) { + ov::Tensor input_tensor; + + if (inputs[i].get_element_type() == ov::element::f32) { + input_tensor = fillTensorRandom(inputs[i]); + } else if (inputs[i].get_element_type() == ov::element::f16) { + input_tensor = fillTensorRandom(inputs[i]); + } else if (inputs[i].get_element_type() == ov::element::i32) { + input_tensor = fillTensorRandom(inputs[i]); + } else if (inputs[i].get_element_type() == ov::element::u8) { + input_tensor = fillTensorRandom(inputs[i]); + } else if (inputs[i].get_element_type() == ov::element::i8) { + input_tensor = fillTensorRandom(inputs[i]); + } else if (inputs[i].get_element_type() == ov::element::u16) { + input_tensor = fillTensorRandom(inputs[i]); + } else if (inputs[i].get_element_type() == ov::element::i16) { + input_tensor = fillTensorRandom(inputs[i]); + } else { + throw std::logic_error( + "Input precision is not supported for " + inputs[i].get_element_type().get_type_name()); + } + infer_request.set_input_tensor(i, input_tensor); + } +} + + +/** + * @brief Fill InferRequest blobs with random values or image information (OV API 1) */ void fillBlobs(InferenceEngine::InferRequest inferRequest, const InferenceEngine::ConstInputsDataMap &inputsInfo, - const size_t &batchSize); - -/** - * @brief Fill InferRequest tensors with random values or image information - */ -void fillTensors(ov::InferRequest &infer_request, - const std::vector> &inputs); + const size_t &batchSize); \ No newline at end of file diff --git a/tests/lib/src/reshape_utils.cpp b/tests/lib/src/reshape_utils.cpp index 66ce7d2b440..e9b293fa8fe 100644 --- a/tests/lib/src/reshape_utils.cpp +++ b/tests/lib/src/reshape_utils.cpp @@ -1,70 +1,76 @@ -// Copyright (C) 2018-2021 Intel Corporation +// Copyright (C) 2018-2022 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // #include "common_utils.h" #include "reshape_utils.h" -#include - -using namespace InferenceEngine; /** * @brief Parse data shapes for model */ -std::map> parseDataShapes(const std::string& shapeString) { - std::map> data_shapes; - // Parse input parameter string - std::vector inputsShapes = split(shapeString, '&'); +std::map> parseDataShapes(const std::string &shapeString) { + std::map> data_shapes; + // Parse input parameter string + std::vector inputsShapes = split(shapeString, '&'); - for (int i = 0; i < inputsShapes.size(); i++) { - std::vector curLayout = split(inputsShapes[i], ':'); + for (int i = 0; i < inputsShapes.size(); i++) { + std::vector curLayout = split(inputsShapes[i], '*'); - std::string curLayoutName = curLayout.at(0); - std::vector shape; + std::string curLayoutName = curLayout.at(0); + std::vector shape; - for (auto& dim : split(curLayout.at(1), ',')) - shape.emplace_back(std::stoi(dim)); - - data_shapes[curLayoutName] = shape; - } - return data_shapes; + try { + for (auto &dim: split(curLayout.at(1), ',')) + shape.emplace_back(std::stoi(dim)); + } catch (const std::exception &ex) { + std::cerr << "Parsing data shapes failed with exception:\n" + << ex.what() << "\n"; + } + data_shapes[curLayoutName] = shape; + } + return data_shapes; } /** * @brief Parse input shapes for model reshape */ -std::map parseReshapeShapes(const std::string& shapeString) { - std::map reshape_info; - // Parse input parameter string - std::vector inputsShapes = split(shapeString, '&'); +std::map parseReshapeShapes(const std::string &shapeString) { + std::map reshape_info; + // Parse input parameter string + std::vector inputsShapes = split(shapeString, '&'); - for (int i = 0; i < inputsShapes.size(); i++) { - std::vector curLayout = split(inputsShapes[i], ':'); + for (int i = 0; i < inputsShapes.size(); i++) { + std::vector curLayout = split(inputsShapes[i], '*'); - std::string curLayoutName = curLayout.at(0); - std::vector shape; + std::string curLayoutName = curLayout.at(0); + std::vector shape; - for (auto& dim : split(curLayout.at(1), ',')) { - if (dim == "?" || dim == "-1") { - shape.emplace_back(ov::Dimension::dynamic()); - } - else { - const std::string range_divider = ".."; - size_t range_index = dim.find(range_divider); - if (range_index != std::string::npos) { - std::string min = dim.substr(0, range_index); - std::string max = dim.substr(range_index + range_divider.length()); - shape.emplace_back(ov::Dimension(std::stoi(min), std::stoi(max))); - } else { - shape.emplace_back(ov::Dimension(std::stoi(dim))); + for (auto& dim : split(curLayout.at(1), ',')) { + try { + if (dim == "?" || dim == "-1") { + shape.emplace_back(ov::Dimension::dynamic()); + } + else { + const std::string range_divider = ".."; + size_t range_index = dim.find(range_divider); + if (range_index != std::string::npos) { + std::string min = dim.substr(0, range_index); + std::string max = dim.substr(range_index + range_divider.length()); + shape.emplace_back(ov::Dimension(std::stoi(min), std::stoi(max))); + } else { + shape.emplace_back(ov::Dimension(std::stoi(dim))); + } + } + } catch (const std::exception &ex) { + std::cerr << "Parsing reshape shapes failed with exception:\n" + << ex.what() << "\n"; + } } - } + reshape_info[curLayoutName] = ov::PartialShape(shape); } - reshape_info[curLayoutName] = ov::PartialShape(shape); - } - return reshape_info; + return reshape_info; } @@ -72,7 +78,7 @@ std::map parseReshapeShapes(const std::string& sh * @brief Split input string using specified delimiter. Return vector with input tensor information */ -std::vector split(const std::string& s, char delim) { +std::vector split(const std::string &s, char delim) { std::vector result; std::stringstream ss(s); std::string item; @@ -87,18 +93,77 @@ std::vector split(const std::string& s, char delim) { /** * @brief Reshape blobs with dynamic shapes with static information from data shape */ -void setStaticShapesBlobs(InferenceEngine::InferRequest inferRequest, - const InferenceEngine::ConstInputsDataMap& inputsInfo, +void setBlobsStaticShapes(InferenceEngine::InferRequest inferRequest, + const InferenceEngine::ConstInputsDataMap &inputsInfo, std::map> dataShape) { - for (const ConstInputsDataMap::value_type& item : inputsInfo) { - Blob::Ptr inputBlob = inferRequest.GetBlob(item.first); + for (const InferenceEngine::ConstInputsDataMap::value_type &item : inputsInfo) { + InferenceEngine::Blob::Ptr inputBlob = inferRequest.GetBlob(item.first); - if (dataShape.count(item.first)) { - SizeVector newInputShape; - for (size_t i = 0; i < dataShape[item.first].size(); i++) { - newInputShape.emplace_back(dataShape[item.first][i]); - } - inputBlob->setShape(newInputShape); + if (dataShape.count(item.first)) { + InferenceEngine::SizeVector newInputShape; + for (size_t i = 0; i < dataShape[item.first].size(); i++) { + newInputShape.emplace_back(dataShape[item.first][i]); + } + inputBlob->setShape(newInputShape); + } + } +} + + +/** + * @brief Getting tensor shapes. If tensor is dynamic, static shape from data info will be returned. + */ +ov::Shape getTensorStaticShape(ov::Output &input, + std::map> dataShape) { + std::string name; + try { + name = input.get_any_name(); + } catch (const ov::Exception &iex) { + // Attempt to get a name for a Tensor without names + } + + ov::Shape inputShape; + + if (!input.get_partial_shape().is_dynamic()) { + return input.get_shape(); + } + else if (dataShape.count(name)) { + for (size_t j = 0; j < dataShape[name].size(); j++) + inputShape.emplace_back(dataShape[name][j]); + } + else { + throw std::logic_error("Please provide static shape for " + name + "input using -data_shapes argument!"); + } + return inputShape; +} + + +/** + * @brief Fill infer_request tensors with random values. The model shape is set separately. (OV API 2) + */ +void fillTensorsWithSpecifiedShape(ov::InferRequest& infer_request, std::vector> &inputs, + std::map> dataShape) { + for (size_t i = 0; i < inputs.size(); i++) { + ov::Tensor input_tensor; + ov::Shape inputShape = getTensorStaticShape(inputs[i], dataShape); + + if (inputs[i].get_element_type() == ov::element::f32) { + input_tensor = fillTensorRandomDynamic(inputs[i], inputShape); + } else if (inputs[i].get_element_type() == ov::element::f16) { + input_tensor = fillTensorRandomDynamic(inputs[i], inputShape); + } else if (inputs[i].get_element_type() == ov::element::i32) { + input_tensor = fillTensorRandomDynamic(inputs[i], inputShape); + } else if (inputs[i].get_element_type() == ov::element::u8) { + input_tensor = fillTensorRandomDynamic(inputs[i], inputShape); + } else if (inputs[i].get_element_type() == ov::element::i8) { + input_tensor = fillTensorRandomDynamic(inputs[i], inputShape); + } else if (inputs[i].get_element_type() == ov::element::u16) { + input_tensor = fillTensorRandomDynamic(inputs[i], inputShape); + } else if (inputs[i].get_element_type() == ov::element::i16) { + input_tensor = fillTensorRandomDynamic(inputs[i], inputShape); + } else { + throw std::logic_error("Input precision is not supported for " + inputs[i].get_element_type().get_type_name()); + } + infer_request.set_input_tensor(i, input_tensor); } - } } \ No newline at end of file diff --git a/tests/lib/src/reshape_utils.h b/tests/lib/src/reshape_utils.h index a58fac7813a..8001531bf75 100644 --- a/tests/lib/src/reshape_utils.h +++ b/tests/lib/src/reshape_utils.h @@ -1,36 +1,66 @@ -// Copyright (C) 2018-2021 Intel Corporation +// Copyright (C) 2018-2022 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // #pragma once #include +#include -using namespace InferenceEngine; + +/** + * @brief Fill InferenceEngine tensor with random values. The model shape is set separately. + */ +template +ov::Tensor fillTensorRandomDynamic(ov::Output &input, ov::Shape shape) { + ov::Tensor tensor {input.get_element_type(), shape}; + std::vectorvalues(ov::shape_size(shape)); + + for (size_t i = 0; i < values.size(); ++i) { + values[i] = 1 + static_cast(rand()) / (static_cast(RAND_MAX / (std::numeric_limits::max() - 1))); + } + std::memcpy(tensor.data(), values.data(), sizeof(T) * values.size()); + + return tensor; +} /** * @brief Split input string using specified delimiter. Return vector with input tensor information */ -std::vector split(const std::string& s, char delim); +std::vector split(const std::string &s, char delim); /** * @brief Parse input shapes for model reshape */ -std::map parseReshapeShapes(const std::string& shapeString); +std::map parseReshapeShapes(const std::string &shapeString); /** * @brief Parse data shapes for model */ -std::map> parseDataShapes(const std::string& shapeString); +std::map> parseDataShapes(const std::string &shapeString); + + +/** + * @brief Getting tensor shapes. If tensor is dynamic, static shape from data info will be returned. + */ +ov::Shape getTensorStaticShape(ov::Output &input, + std::map> dataShape); /** * @brief Reshape blobs with dynamic shapes with static information from data shape */ -void setStaticShapesBlobs(InferenceEngine::InferRequest inferRequest, - const InferenceEngine::ConstInputsDataMap& inputsInfo, - std::map> dataShape); \ No newline at end of file +void setBlobsStaticShapes(InferenceEngine::InferRequest inferRequest, + const InferenceEngine::ConstInputsDataMap &inputsInfo, + std::map> dataShape); + + +/** + * @brief Fill infer_request tensors with random values. The model shape is set separately. (OV API 2) + */ +void fillTensorsWithSpecifiedShape(ov::InferRequest& infer_request, std::vector> &inputs, + std::map> dataShape); \ No newline at end of file diff --git a/tests/time_tests/include/timetests_helper/timer.h b/tests/time_tests/include/timetests_helper/timer.h index 0bda9c8805a..68dca76445f 100644 --- a/tests/time_tests/include/timetests_helper/timer.h +++ b/tests/time_tests/include/timetests_helper/timer.h @@ -8,24 +8,24 @@ #include namespace TimeTest { -using time_point = std::chrono::high_resolution_clock::time_point; + using time_point = std::chrono::high_resolution_clock::time_point; /** Encapsulate time measurements. Object of a class measures time at start and finish of object's life cycle. When destroyed, reports duration. */ -class Timer { -private: - std::string name; - time_point start_time; + class Timer { + private: + std::string name; + time_point start_time; -public: - /// Constructs Timer object and measures start time. - Timer(const std::string &timer_name); + public: + /// Constructs Timer object and measures start time. + Timer(const std::string &timer_name); - /// Destructs Timer object, measures duration and reports it. - ~Timer(); -}; + /// Destructs Timer object, measures duration and reports it. + ~Timer(); + }; #define SCOPED_TIMER(timer_name) TimeTest::Timer timer_name(#timer_name); diff --git a/tests/time_tests/include/timetests_helper/utils.h b/tests/time_tests/include/timetests_helper/utils.h index 340fcbf7e72..79da8871c42 100644 --- a/tests/time_tests/include/timetests_helper/utils.h +++ b/tests/time_tests/include/timetests_helper/utils.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include @@ -22,17 +23,32 @@ std::string fileExt(const std::string &filename) { } /** - * @brief Function that enables Latency performance hint for specified device. + * @brief Function that enables Latency performance hint for specified device (OV API 1) */ void setPerformanceConfig(InferenceEngine::Core ie, const std::string &device) { - std::vector supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)); + std::vector supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)); - if (std::find(supported_config_keys.begin(), supported_config_keys.end(), "PERFORMANCE_HINT") == - supported_config_keys.end()) { - std::cerr << "Device " << device << " doesn't support config key 'PERFORMANCE_HINT'!\n" - << "Performance config was not set."; - } - else - ie.SetConfig({{CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(LATENCY)}}, device); + if (std::find(supported_config_keys.begin(), supported_config_keys.end(), "PERFORMANCE_HINT") == + supported_config_keys.end()) { + std::cerr << "Device " << device << " doesn't support config key 'PERFORMANCE_HINT'!\n" + << "Performance config was not set."; + } + else + ie.SetConfig({{CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(LATENCY)}}, device); +} + +/** + * @brief Function that enables Latency performance hint for specified device (OV API 2) + */ +void setPerformanceConfig(ov::Core ie, const std::string &device) { + std::vector supported_config_keys = ie.get_property(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)); + + if (std::find(supported_config_keys.begin(), supported_config_keys.end(), "PERFORMANCE_HINT") == + supported_config_keys.end()) { + std::cerr << "Device " << device << " doesn't support config key 'PERFORMANCE_HINT'!\n" + << "Performance config was not set."; + } + else + ie.set_property(device, {{CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(LATENCY)}}); } } \ No newline at end of file diff --git a/tests/time_tests/src/timetests/timetest_infer.cpp b/tests/time_tests/src/timetests/timetest_infer.cpp index 2998aa691f4..beedd034129 100644 --- a/tests/time_tests/src/timetests/timetest_infer.cpp +++ b/tests/time_tests/src/timetests/timetest_infer.cpp @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // -#include #include #include "common_utils.h" @@ -10,8 +9,6 @@ #include "timetests_helper/timer.h" #include "timetests_helper/utils.h" -using namespace InferenceEngine; - /** * @brief Function that contain executable pipeline which will be called from @@ -21,96 +18,94 @@ using namespace InferenceEngine; int runPipeline(const std::string &model, const std::string &device, const bool isCacheEnabled, std::map reshapeShapes, std::map> dataShapes) { - auto pipeline = [](const std::string &model, const std::string &device, const bool isCacheEnabled, - std::map reshapeShapes, - std::map> dataShapes) { - Core ie; - CNNNetwork cnnNetwork; - ExecutableNetwork exeNetwork; - InferRequest inferRequest; - size_t batchSize = 0; + auto pipeline = [](const std::string &model, const std::string &device, const bool isCacheEnabled, + std::map reshapeShapes, + std::map> dataShapes) { + InferenceEngine::Core ie; + InferenceEngine::CNNNetwork cnnNetwork; + InferenceEngine::ExecutableNetwork exeNetwork; + InferenceEngine::InferRequest inferRequest; + size_t batchSize = 0; - bool reshape = false; - if (!reshapeShapes.empty()) { - reshape = true; - } + bool reshape = false; + if (!reshapeShapes.empty()) { + reshape = true; + } - // first_inference_latency = time_to_inference + first_inference - { - SCOPED_TIMER(time_to_inference); - { - SCOPED_TIMER(load_plugin); - ie.GetVersions(device); - } - // Set performance and model cache configs - TimeTest::setPerformanceConfig(ie, device); - if (isCacheEnabled) - ie.SetConfig({{CONFIG_KEY(CACHE_DIR), "models_cache"}}); - { - SCOPED_TIMER(create_exenetwork); - if (!isCacheEnabled) { - if (TimeTest::fileExt(model) == "blob") { - SCOPED_TIMER(import_network); - exeNetwork = ie.ImportNetwork(model, device); - } - else { + // first_inference_latency = time_to_inference + first_inference + { + SCOPED_TIMER(time_to_inference); { - SCOPED_TIMER(read_network); - cnnNetwork = ie.ReadNetwork(model); - batchSize = cnnNetwork.getBatchSize(); - } - if (reshape) { - { - SCOPED_TIMER(reshape); - cnnNetwork.reshape(reshapeShapes); - } + SCOPED_TIMER(load_plugin); + TimeTest::setPerformanceConfig(ie, device); + ie.GetVersions(device); + + if (isCacheEnabled) + ie.SetConfig({{CONFIG_KEY(CACHE_DIR), "models_cache"}}); } { - SCOPED_TIMER(load_network); - exeNetwork = ie.LoadNetwork(cnnNetwork, device); + SCOPED_TIMER(create_exenetwork); + if (!isCacheEnabled) { + if (TimeTest::fileExt(model) == "blob") { + SCOPED_TIMER(import_network); + exeNetwork = ie.ImportNetwork(model, device); + } + else { + { + SCOPED_TIMER(read_network); + cnnNetwork = ie.ReadNetwork(model); + batchSize = cnnNetwork.getBatchSize(); + } + if (reshape) { + { + SCOPED_TIMER(reshape); + cnnNetwork.reshape(reshapeShapes); + } + } + { + SCOPED_TIMER(load_network); + exeNetwork = ie.LoadNetwork(cnnNetwork, device); + } + } + } + else { + SCOPED_TIMER(load_network_cache); + exeNetwork = ie.LoadNetwork(model, device); + } } - } + inferRequest = exeNetwork.CreateInferRequest(); } - else { - SCOPED_TIMER(load_network_cache); - exeNetwork = ie.LoadNetwork(model, device); + { + SCOPED_TIMER(first_inference); + { + SCOPED_TIMER(fill_inputs); + const InferenceEngine::ConstInputsDataMap inputsInfo(exeNetwork.GetInputsInfo()); + batchSize = batchSize != 0 ? batchSize : 1; + if (reshape) { + setBlobsStaticShapes(inferRequest, inputsInfo, dataShapes); + fillBlobs(inferRequest, inputsInfo, batchSize); + } else { + fillBlobs(inferRequest, inputsInfo, batchSize); + } + } + inferRequest.Infer(); } - } - inferRequest = exeNetwork.CreateInferRequest(); + }; + + try { + pipeline(model, device, isCacheEnabled, reshapeShapes, dataShapes); + } catch (const InferenceEngine::Exception &iex) { + std::cerr + << "Inference Engine pipeline failed with Inference Engine exception:\n" + << iex.what(); + return 1; + } catch (const std::exception &ex) { + std::cerr << "Inference Engine pipeline failed with exception:\n" + << ex.what(); + return 2; + } catch (...) { + std::cerr << "Inference Engine pipeline failed\n"; + return 3; } - { - SCOPED_TIMER(first_inference); - { - SCOPED_TIMER(fill_inputs); - - const InferenceEngine::ConstInputsDataMap inputsInfo(exeNetwork.GetInputsInfo()); - batchSize = batchSize != 0 ? batchSize : 1; - - if (reshape) { - setStaticShapesBlobs(inferRequest, inputsInfo, dataShapes); - fillBlobs(inferRequest, inputsInfo, batchSize); - } else { - fillBlobs(inferRequest, inputsInfo, batchSize); - } - } - inferRequest.Infer(); - } - }; - - try { - pipeline(model, device, isCacheEnabled, reshapeShapes, dataShapes); - } catch (const InferenceEngine::Exception &iex) { - std::cerr - << "Inference Engine pipeline failed with Inference Engine exception:\n" - << iex.what(); - return 1; - } catch (const std::exception &ex) { - std::cerr << "Inference Engine pipeline failed with exception:\n" - << ex.what(); - return 2; - } catch (...) { - std::cerr << "Inference Engine pipeline failed\n"; - return 3; - } - return 0; + return 0; } \ No newline at end of file diff --git a/tests/time_tests/src/timetests/timetest_infer_api_2.cpp b/tests/time_tests/src/timetests/timetest_infer_api_2.cpp new file mode 100644 index 00000000000..c7ec3a7a1fd --- /dev/null +++ b/tests/time_tests/src/timetests/timetest_infer_api_2.cpp @@ -0,0 +1,109 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include + +#include + +#include "common_utils.h" +#include "reshape_utils.h" +#include "timetests_helper/timer.h" +#include "timetests_helper/utils.h" + + +/** + * @brief Function that contain executable pipeline which will be called from + * main(). The function should not throw any exceptions and responsible for + * handling it by itself. + */ +int runPipeline(const std::string &model, const std::string &device, const bool isCacheEnabled, + std::map reshapeShapes, + std::map> dataShapes) { + auto pipeline = [](const std::string &model, const std::string &device, const bool isCacheEnabled, + std::map reshapeShapes, + std::map> dataShapes) { + ov::Core ie; + std::shared_ptr cnnNetwork; + ov::CompiledModel exeNetwork; + ov::InferRequest inferRequest; + + bool reshape = false; + if (!reshapeShapes.empty()) { + reshape = true; + } + + // first_inference_latency = time_to_inference + first_inference + { + SCOPED_TIMER(time_to_inference); + { + SCOPED_TIMER(load_plugin); + TimeTest::setPerformanceConfig(ie, device); + ie.get_versions(device); + + if (isCacheEnabled) + ie.set_property({{CONFIG_KEY(CACHE_DIR), "models_cache"}}); + } + { + SCOPED_TIMER(create_exenetwork); + if (!isCacheEnabled) { + if (TimeTest::fileExt(model) == "blob") { + SCOPED_TIMER(import_network); + std::ifstream streamModel{model}; + exeNetwork = ie.import_model(streamModel, device); + } + else { + { + SCOPED_TIMER(read_network); + cnnNetwork = ie.read_model(model); + } + if (reshape) { + { + SCOPED_TIMER(reshape); + cnnNetwork->reshape(reshapeShapes); + } + } + { + SCOPED_TIMER(load_network); + exeNetwork = ie.compile_model(cnnNetwork, device); + } + } + } + else { + SCOPED_TIMER(load_network_cache); + exeNetwork = ie.compile_model(model, device); + } + } + inferRequest = exeNetwork.create_infer_request(); + } + { + SCOPED_TIMER(first_inference); + { + SCOPED_TIMER(fill_inputs); + std::vector> inputs = exeNetwork.inputs(); + if (reshape) { + fillTensorsWithSpecifiedShape(inferRequest, inputs,dataShapes); + } else { + fillTensors(inferRequest, inputs); + } + } + inferRequest.infer(); + } + }; + + try { + pipeline(model, device, isCacheEnabled, reshapeShapes, dataShapes); + } catch (const ov::Exception &iex) { + std::cerr + << "Inference Engine pipeline failed with Inference Engine exception:\n" + << iex.what(); + return 1; + } catch (const std::exception &ex) { + std::cerr << "Inference Engine pipeline failed with exception:\n" + << ex.what(); + return 2; + } catch (...) { + std::cerr << "Inference Engine pipeline failed\n"; + return 3; + } + return 0; +} \ No newline at end of file diff --git a/tests/time_tests/src/timetests_helper/cli.h b/tests/time_tests/src/timetests_helper/cli.h index febf95470ee..d79a4d88a85 100644 --- a/tests/time_tests/src/timetests_helper/cli.h +++ b/tests/time_tests/src/timetests_helper/cli.h @@ -11,40 +11,40 @@ /// @brief message for help argument static const char help_message[] = - "Print a usage message."; + "Print a usage message."; /// @brief message for model argument static const char model_message[] = - "Required. Path to an .xml/.onnx file with a trained model or to " - "a .blob files with a trained compiled model."; + "Required. Path to an .xml/.onnx file with a trained model or to " + "a .blob files with a trained compiled model."; /// @brief message for target device argument static const char target_device_message[] = - "Required. Specify a target device to infer on. \n" - "Use \"-d HETERO:\" format to specify HETERO " - "plugin. \n" - "Use \"-d MULTI:\" format to specify MULTI " - "plugin. \n" - "The application looks for a suitable plugin for the specified device."; + "Required. Specify a target device to infer on. \n" + "Use \"-d HETERO:\" format to specify HETERO " + "plugin. \n" + "Use \"-d MULTI:\" format to specify MULTI " + "plugin. \n" + "The application looks for a suitable plugin for the specified device."; /// @brief message for cache argument static const char model_cache_message[] = - "Not required. Use this key to run timetests with models caching. \n" - "TimeInfer executable should be run twice - the second run will use cache prepared from first run."; + "Not required. Use this key to run timetests with models caching. \n" + "TimeInfer executable should be run twice - the second run will use cache prepared from first run."; /// @brief message for shapes argument static const char reshape_shapes_message[] = - "Not required. Use this key to run timetests with reshape. \n" - "Example: 'input:1..2 3 100 100'. Use '&' delimiter for several inputs. Example: 'input1:1..2 100&input2:1..2 100' "; + "Not required. Use this key to run timetests with reshape. \n" + "Example: 'input*1..2 3 100 100'. Use '&' delimiter for several inputs. Example: 'input1*1..2 100&input2*1..2 100' "; /// @brief message for shapes argument static const char data_shapes_message[] = - "Not required. Use this key to run timetests with reshape. Used with 'reshape_shapes' arg. \n" - "Only static shapes for data. Example: 'input:1 3 100 100'. Use '&' delimiter for several inputs. Example: 'input1:1 100&input2:1 100' "; + "Not required. Use this key to run timetests with reshape. Used with 'reshape_shapes' arg. \n" + "Only static shapes for data. Example: 'input*1 3 100 100'. Use '&' delimiter for several inputs. Example: 'input1*1 100&input2*1 100' "; /// @brief message for statistics path argument static const char statistics_path_message[] = - "Required. Path to a file to write statistics."; + "Required. Path to a file to write statistics."; /// @brief Define flag for showing help message
DEFINE_bool(h, false, help_message); @@ -80,15 +80,15 @@ DEFINE_string(s, "", statistics_path_message); * @brief This function show a help message */ static void showUsage() { - std::cout << std::endl; - std::cout << "TimeInfer [OPTION]" << std::endl; - std::cout << "Options:" << std::endl; - std::cout << std::endl; - std::cout << " -h, --help " << help_message << std::endl; - std::cout << " -m \"\" " << model_message << std::endl; - std::cout << " -d \"\" " << target_device_message << std::endl; - std::cout << " -s \"\" " << statistics_path_message << std::endl; - std::cout << " -c " << model_cache_message << std::endl; - std::cout << " -reshape_shapes " << reshape_shapes_message << std::endl; - std::cout << " -data_shapes " << data_shapes_message << std::endl; + std::cout << std::endl; + std::cout << "TimeInfer [OPTION]" << std::endl; + std::cout << "Options:" << std::endl; + std::cout << std::endl; + std::cout << " -h, --help " << help_message << std::endl; + std::cout << " -m \"\" " << model_message << std::endl; + std::cout << " -d \"\" " << target_device_message << std::endl; + std::cout << " -s \"\" " << statistics_path_message << std::endl; + std::cout << " -c " << model_cache_message << std::endl; + std::cout << " -reshape_shapes " << reshape_shapes_message << std::endl; + std::cout << " -data_shapes " << data_shapes_message << std::endl; } \ No newline at end of file diff --git a/tests/time_tests/src/timetests_helper/main.cpp b/tests/time_tests/src/timetests_helper/main.cpp index b775cfc3237..84822a96455 100644 --- a/tests/time_tests/src/timetests_helper/main.cpp +++ b/tests/time_tests/src/timetests_helper/main.cpp @@ -18,29 +18,29 @@ int runPipeline(const std::string &model, const std::string &device, const bool * @brief Parses command line and check required arguments */ bool parseAndCheckCommandLine(int argc, char **argv) { - gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); - if (FLAGS_help || FLAGS_h) { - showUsage(); - return false; - } + gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); + if (FLAGS_help || FLAGS_h) { + showUsage(); + return false; + } - if (FLAGS_m.empty()) - throw std::logic_error( - "Model is required but not set. Please set -m option."); + if (FLAGS_m.empty()) + throw std::logic_error( + "Model is required but not set. Please set -m option."); - if (FLAGS_d.empty()) - throw std::logic_error( - "Device is required but not set. Please set -d option."); + if (FLAGS_d.empty()) + throw std::logic_error( + "Device is required but not set. Please set -d option."); - if (FLAGS_s.empty()) - throw std::logic_error( - "Statistics file path is required but not set. Please set -s option."); + if (FLAGS_s.empty()) + throw std::logic_error( + "Statistics file path is required but not set. Please set -s option."); - if (!FLAGS_reshape_shapes.empty() && FLAGS_data_shapes.empty()) - throw std::logic_error( - "Data shapes is required for reshape shapes argument. Please set -data_shapes option."); + if (!FLAGS_reshape_shapes.empty() && FLAGS_data_shapes.empty()) + throw std::logic_error( + "Data shapes is required for reshape shapes argument. Please set -data_shapes option."); - return true; + return true; } /** @@ -48,22 +48,22 @@ bool parseAndCheckCommandLine(int argc, char **argv) { */ int _runPipeline(std::map dynamicShapes, std::map> staticShapes) { - SCOPED_TIMER(full_run); - return runPipeline(FLAGS_m, FLAGS_d, FLAGS_c, dynamicShapes, staticShapes); + SCOPED_TIMER(full_run); + return runPipeline(FLAGS_m, FLAGS_d, FLAGS_c, dynamicShapes, staticShapes); } /** * @brief Main entry point */ int main(int argc, char **argv) { - if (!parseAndCheckCommandLine(argc, argv)) - return -1; + if (!parseAndCheckCommandLine(argc, argv)) + return -1; - auto dynamicShapes = parseReshapeShapes(FLAGS_reshape_shapes); - auto staticShapes = parseDataShapes(FLAGS_data_shapes); + auto dynamicShapes = parseReshapeShapes(FLAGS_reshape_shapes); + auto staticShapes = parseDataShapes(FLAGS_data_shapes); - auto status = _runPipeline(dynamicShapes, staticShapes); - StatisticsWriter::Instance().setFile(FLAGS_s); - StatisticsWriter::Instance().write(); - return status; + auto status = _runPipeline(dynamicShapes, staticShapes); + StatisticsWriter::Instance().setFile(FLAGS_s); + StatisticsWriter::Instance().write(); + return status; } \ No newline at end of file diff --git a/tests/time_tests/src/timetests_helper/statistics_writer.h b/tests/time_tests/src/timetests_helper/statistics_writer.h index 03cafc8dcce..8bf4c49e0e9 100644 --- a/tests/time_tests/src/timetests_helper/statistics_writer.h +++ b/tests/time_tests/src/timetests_helper/statistics_writer.h @@ -22,62 +22,62 @@ */ class StatisticsWriter { private: - std::ofstream statistics_file; + std::ofstream statistics_file; - std::map> time_structure; // timer_name, - std::vector time_struct_order; - int tab_count = 0; + std::map> time_structure; // timer_name, + std::vector time_struct_order; + int tab_count = 0; - StatisticsWriter() = default; - StatisticsWriter(const StatisticsWriter &) = delete; - StatisticsWriter &operator=(const StatisticsWriter &) = delete; + StatisticsWriter() = default; + StatisticsWriter(const StatisticsWriter &) = delete; + StatisticsWriter &operator=(const StatisticsWriter &) = delete; public: - /** - * @brief Creates StatisticsWriter singleton object - */ - static StatisticsWriter &Instance() { +/** + * @brief Creates StatisticsWriter singleton object + */ +static StatisticsWriter &Instance() { static StatisticsWriter writer; return writer; - } +} - /** - * @brief Specifies, opens and validates statistics path for writing - */ - void setFile(const std::string &statistics_path) { +/** + * @brief Specifies, opens and validates statistics path for writing + */ +void setFile(const std::string &statistics_path) { statistics_file.open(statistics_path); if (!statistics_file.good()) { - std::stringstream err; - err << "Statistic file \"" << statistics_path - << "\" can't be used for writing"; - throw std::runtime_error(err.str()); + std::stringstream err; + err << "Statistic file \"" << statistics_path + << "\" can't be used for writing"; + throw std::runtime_error(err.str()); } - } +} - /** - * @brief Compute order for statistics operations. - */ - void addTimer(const std::string name) { +/** + * @brief Compute order for statistics operations. + */ +void addTimer(const std::string name) { time_struct_order.push_back(name); tab_count++; - } +} - void deleteTimer(const std::pair &record) { +void deleteTimer(const std::pair &record) { tab_count--; time_structure[record.first] = std::make_pair(tab_count, record.second); - } +} - /** - * @brief Writes provided statistics in YAML format. - */ - void write() { +/** + * @brief Writes provided statistics in YAML format. + */ +void write() { if (!statistics_file) - throw std::runtime_error("Statistic file path isn't set"); + throw std::runtime_error("Statistic file path isn't set"); for (auto& timer: time_struct_order) { - std::string tabs = std::string(TAB* time_structure[timer].first, ' '); - statistics_file << tabs << "- " << timer << ":" << '\n' - << tabs << " " << "- " << time_structure[timer].second << '\n'; + std::string tabs = std::string(TAB* time_structure[timer].first, ' '); + statistics_file << tabs << "- " << timer << ":" << '\n' + << tabs << " " << "- " << time_structure[timer].second << '\n'; } statistics_file << "---" << '\n' << "measurement_unit: microsecs"; - } +} }; diff --git a/tests/time_tests/src/timetests_helper/timer.cpp b/tests/time_tests/src/timetests_helper/timer.cpp index 9b02fb3d266..cd8f570c676 100644 --- a/tests/time_tests/src/timetests_helper/timer.cpp +++ b/tests/time_tests/src/timetests_helper/timer.cpp @@ -14,18 +14,17 @@ using time_point = std::chrono::high_resolution_clock::time_point; namespace TimeTest { - Timer::Timer(const std::string &timer_name) { - name = timer_name; - start_time = std::chrono::high_resolution_clock::now(); - StatisticsWriter::Instance().addTimer(name); + name = timer_name; + start_time = std::chrono::high_resolution_clock::now(); + StatisticsWriter::Instance().addTimer(name); } Timer::~Timer() { - float duration = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - start_time) - .count(); - StatisticsWriter::Instance().deleteTimer({name, duration}); + float duration = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - start_time) + .count(); + StatisticsWriter::Instance().deleteTimer({name, duration}); } } // namespace TimeTest \ No newline at end of file