From 52937967bb6317383287d3d8866ec1e8b7d033c9 Mon Sep 17 00:00:00 2001 From: Valentin Dymchishin Date: Mon, 28 Mar 2022 12:51:53 +0300 Subject: [PATCH] Add dynamism in memory tests (API 2) (#10589) --- tests/memory_tests/CMakeLists.txt | 1 + .../memory_tests_helper/memory_counter.h | 2 +- .../src/memory_tests/CMakeLists.txt | 2 +- .../src/memory_tests/memtest_infer.cpp | 10 +++- .../src/memory_tests/memtest_infer_api_2.cpp | 57 +++++++++++++------ .../src/memory_tests_helper/CMakeLists.txt | 2 +- .../src/memory_tests_helper/cli.h | 32 ++++++++--- .../src/memory_tests_helper/main.cpp | 19 +++++-- 8 files changed, 89 insertions(+), 36 deletions(-) diff --git a/tests/memory_tests/CMakeLists.txt b/tests/memory_tests/CMakeLists.txt index 57be14b1131..d07250daa64 100644 --- a/tests/memory_tests/CMakeLists.txt +++ b/tests/memory_tests/CMakeLists.txt @@ -29,3 +29,4 @@ add_subdirectory(src) install(DIRECTORY test_runner/ DESTINATION tests/memory_tests/test_runner COMPONENT tests EXCLUDE_FROM_ALL) install(DIRECTORY .automation/ DESTINATION tests/memory_tests/test_runner/.automation COMPONENT tests EXCLUDE_FROM_ALL) install(DIRECTORY scripts/ DESTINATION tests/memory_tests/scripts COMPONENT tests EXCLUDE_FROM_ALL) +install(DIRECTORY ../utils/ DESTINATION tests/utils COMPONENT tests EXCLUDE_FROM_ALL) diff --git a/tests/memory_tests/include/memory_tests_helper/memory_counter.h b/tests/memory_tests/include/memory_tests_helper/memory_counter.h index 63c479adaad..17c86eafe5f 100644 --- a/tests/memory_tests/include/memory_tests_helper/memory_counter.h +++ b/tests/memory_tests/include/memory_tests_helper/memory_counter.h @@ -22,6 +22,6 @@ public: MemoryCounter(const std::string &mem_counter_name); }; -#define MEMORY_SNAPSHOT(mem_counter_name) MemoryTest::MemoryCounter (#mem_counter_name); +#define MEMORY_SNAPSHOT(mem_counter_name) MemoryTest::MemoryCounter mem_counter_name(#mem_counter_name); } // namespace MemoryTest diff --git a/tests/memory_tests/src/memory_tests/CMakeLists.txt b/tests/memory_tests/src/memory_tests/CMakeLists.txt index 0c6fdc5010b..cfe2093c030 100644 --- a/tests/memory_tests/src/memory_tests/CMakeLists.txt +++ b/tests/memory_tests/src/memory_tests/CMakeLists.txt @@ -15,7 +15,7 @@ foreach(test_source ${tests}) get_filename_component(test_name ${test_source} NAME_WE) add_executable(${test_name} ${test_source}) - target_link_libraries(${test_name} PRIVATE memory_tests_helper tests_shared_lib) + target_link_libraries(${test_name} PRIVATE tests_shared_lib memory_tests_helper) add_dependencies(memory_tests ${test_name}) diff --git a/tests/memory_tests/src/memory_tests/memtest_infer.cpp b/tests/memory_tests/src/memory_tests/memtest_infer.cpp index 17d601d761f..709d62cd82f 100644 --- a/tests/memory_tests/src/memory_tests/memtest_infer.cpp +++ b/tests/memory_tests/src/memory_tests/memtest_infer.cpp @@ -15,8 +15,12 @@ * main(). The function should not throw any exceptions and responsible for * handling it by itself. */ -int runPipeline(const std::string &model, const std::string &device) { - auto pipeline = [](const std::string &model, const std::string &device) { +int runPipeline(const std::string &model, const std::string &device, + std::map reshapeShapes, + std::map> dataShapes) { + auto pipeline = [](const std::string &model, const std::string &device, + std::map reshapeShapes, + std::map> dataShapes) { InferenceEngine::Core ie; InferenceEngine::CNNNetwork cnnNetwork; InferenceEngine::ExecutableNetwork exeNetwork; @@ -53,7 +57,7 @@ int runPipeline(const std::string &model, const std::string &device) { }; try { - pipeline(model, device); + pipeline(model, device, reshapeShapes, dataShapes); } catch (const InferenceEngine::Exception &iex) { std::cerr << "Inference Engine pipeline failed with Inference Engine exception:\n" diff --git a/tests/memory_tests/src/memory_tests/memtest_infer_api_2.cpp b/tests/memory_tests/src/memory_tests/memtest_infer_api_2.cpp index 8eccc9c210f..2151af0468e 100644 --- a/tests/memory_tests/src/memory_tests/memtest_infer_api_2.cpp +++ b/tests/memory_tests/src/memory_tests/memtest_infer_api_2.cpp @@ -1,15 +1,15 @@ // Copyright (C) 2018-2022 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // +#include -#include -#include #include #include "common_utils.h" +#include "reshape_utils.h" #include "memory_tests_helper/memory_counter.h" #include "memory_tests_helper/utils.h" -#include "openvino/runtime/core.hpp" + /** @@ -17,43 +17,66 @@ * main(). The function should not throw any exceptions and responsible for * handling it by itself. */ -int runPipeline(const std::string &model, const std::string &device) { - auto pipeline = [](const std::string &model, const std::string &device) { +int runPipeline(const std::string &model, const std::string &device, + std::map reshapeShapes, + std::map> dataShapes) { + auto pipeline = [](const std::string &model, const std::string &device, + std::map reshapeShapes, + std::map> dataShapes) { ov::Core ie; - std::shared_ptr network; - ov::CompiledModel compiled_model; - ov::InferRequest infer_request; + std::shared_ptr cnnNetwork; + ov::CompiledModel exeNetwork; + ov::InferRequest inferRequest; + + std::vector> defaultInputs; + + bool reshape = false; + if (!reshapeShapes.empty()) { + reshape = true; + } ie.get_versions(device); MEMORY_SNAPSHOT(load_plugin); if (MemoryTest::fileExt(model) == "blob") { std::ifstream streamModel{model}; - compiled_model = ie.import_model(streamModel, device); + exeNetwork = ie.import_model(streamModel, device); MEMORY_SNAPSHOT(import_network); } else { - network = ie.read_model(model); + cnnNetwork = ie.read_model(model); MEMORY_SNAPSHOT(read_network); - compiled_model = ie.compile_model(network, device); + if (reshape) { + defaultInputs = getCopyOfDefaultInputs(cnnNetwork->inputs()); + cnnNetwork->reshape(reshapeShapes); + MEMORY_SNAPSHOT(reshape); + } + + exeNetwork = ie.compile_model(cnnNetwork, device); MEMORY_SNAPSHOT(load_network); } MEMORY_SNAPSHOT(create_exenetwork); - infer_request = compiled_model.create_infer_request(); + inferRequest = exeNetwork.create_infer_request(); - auto inputs = network->inputs(); - fillTensors(infer_request, inputs); - MEMORY_SNAPSHOT(fill_inputs) + std::vector> inputs = exeNetwork.inputs(); + if (reshape && dataShapes.empty()) { + fillTensors(inferRequest, defaultInputs); + } else if (reshape && !dataShapes.empty()) { + fillTensorsWithSpecifiedShape(inferRequest, inputs, dataShapes); + } else { + fillTensors(inferRequest, inputs); + } + MEMORY_SNAPSHOT(fill_inputs); - infer_request.infer(); + inferRequest.infer(); MEMORY_SNAPSHOT(first_inference); MEMORY_SNAPSHOT(full_run); }; try { - pipeline(model, device); + pipeline(model, device, reshapeShapes, dataShapes); } catch (const InferenceEngine::Exception &iex) { std::cerr << "Inference Engine pipeline failed with Inference Engine exception:\n" diff --git a/tests/memory_tests/src/memory_tests_helper/CMakeLists.txt b/tests/memory_tests/src/memory_tests_helper/CMakeLists.txt index 37d7cd07f34..e9a0bc50028 100644 --- a/tests/memory_tests/src/memory_tests_helper/CMakeLists.txt +++ b/tests/memory_tests/src/memory_tests_helper/CMakeLists.txt @@ -12,4 +12,4 @@ add_subdirectory(${OpenVINO_SOURCE_DIR}/thirdparty/gflags ${CMAKE_CURRENT_BINARY_DIR}/gflags_build EXCLUDE_FROM_ALL) -target_link_libraries(${TARGET_NAME} PUBLIC gflags) +target_link_libraries(${TARGET_NAME} PUBLIC gflags tests_shared_lib) diff --git a/tests/memory_tests/src/memory_tests_helper/cli.h b/tests/memory_tests/src/memory_tests_helper/cli.h index c6ed93db117..dfceff6eab5 100644 --- a/tests/memory_tests/src/memory_tests_helper/cli.h +++ b/tests/memory_tests/src/memory_tests_helper/cli.h @@ -26,6 +26,16 @@ static const char target_device_message[] = "plugin. " "The application looks for a suitable plugin for the specified device."; +/// @brief message for shapes argument +static const char reshape_shapes_message[] = + "Not required. Use this key to run memory tests with reshape. \n" + "Example: 'input*1..2 3 100 100'. Use '&' delimiter for several inputs. Example: 'input1*1..2 100&input2*1..2 100' "; + +/// @brief message for shapes argument +static const char data_shapes_message[] = + "Not required. Use this key to run memory tests with reshape. Used with 'reshape_shapes' arg. \n" + "Only static shapes for data. Example: 'input*1 3 100 100'. Use '&' delimiter for several inputs. Example: 'input1*1 100&input2*1 100' "; + /// @brief message for statistics path argument static const char statistics_path_message[] = "Required. Path to a file to write statistics."; @@ -44,6 +54,14 @@ DEFINE_string(m, "", model_message); /// It is a required parameter DEFINE_string(d, "", target_device_message); +/// @brief Define parameter for set shapes to reshape function
+/// It is a non-required parameter +DEFINE_string(reshape_shapes, "", reshape_shapes_message); + +/// @brief Define parameter for set shapes of the network data
+/// It is a non-required parameter +DEFINE_string(data_shapes, "", data_shapes_message); + /// @brief Define parameter for set path to a file to write statistics
/// It is a required parameter DEFINE_string(s, "", statistics_path_message); @@ -53,13 +71,13 @@ DEFINE_string(s, "", statistics_path_message); */ static void showUsage() { std::cout << std::endl; - std::cout << "TimeTests [OPTION]" << std::endl; + std::cout << "MemoryInfer [OPTION]" << std::endl; std::cout << "Options:" << std::endl; std::cout << std::endl; - std::cout << " -h, --help " << help_message << std::endl; - std::cout << " -m \"\" " << model_message << std::endl; - std::cout << " -d \"\" " << target_device_message - << std::endl; - std::cout << " -s \"\" " << statistics_path_message - << std::endl; + std::cout << " -h, --help " << help_message << std::endl; + std::cout << " -m \"\" " << model_message << std::endl; + std::cout << " -d \"\" " << target_device_message << std::endl; + std::cout << " -s \"\" " << statistics_path_message << std::endl; + std::cout << " -reshape_shapes " << reshape_shapes_message << std::endl; + std::cout << " -data_shapes " << data_shapes_message << std::endl; } diff --git a/tests/memory_tests/src/memory_tests_helper/main.cpp b/tests/memory_tests/src/memory_tests_helper/main.cpp index 2d71137bfaa..357edbb1a4b 100644 --- a/tests/memory_tests/src/memory_tests_helper/main.cpp +++ b/tests/memory_tests/src/memory_tests_helper/main.cpp @@ -4,11 +4,14 @@ #include "cli.h" #include "statistics_writer.h" +#include "reshape_utils.h" #include "memory_tests_helper/memory_counter.h" #include -int runPipeline(const std::string &model, const std::string &device); +int runPipeline(const std::string &model, const std::string &device, + std::map reshapeShapes, + std::map> dataShapes); /** * @brief Parses command line and check required arguments @@ -38,10 +41,11 @@ bool parseAndCheckCommandLine(int argc, char **argv) { /** * @brief Function calls `runPipeline` with mandatory memory values tracking of full run */ -int _runPipeline() { - auto status = runPipeline(FLAGS_m, FLAGS_d); - MEMORY_SNAPSHOT(after_objects_release); - return status; +int _runPipeline(std::map dynamicShapes, + std::map> staticShapes) { + auto status = runPipeline(FLAGS_m, FLAGS_d, dynamicShapes, staticShapes); + MEMORY_SNAPSHOT(after_objects_release); + return status; } /** @@ -51,7 +55,10 @@ int main(int argc, char **argv) { if (!parseAndCheckCommandLine(argc, argv)) return -1; - auto status = _runPipeline(); + auto dynamicShapes = parseReshapeShapes(FLAGS_reshape_shapes); + auto staticShapes = parseDataShapes(FLAGS_data_shapes); + + auto status = _runPipeline(dynamicShapes, staticShapes); StatisticsWriter::Instance().setFile(FLAGS_s); StatisticsWriter::Instance().write(); return status;