From f84a6d97ace26b75ae9d852525a997cbaba32e6a Mon Sep 17 00:00:00 2001 From: Andrey Somsikov Date: Mon, 14 Sep 2020 09:04:49 +0300 Subject: [PATCH] Split time-tests common library (#2173) Split time-tests common library Add a README.md with workflow description. Defined "timetest_" suffix for all time tests. Applied clang-format-9 and added a README.md Co-authored-by: Alina Alborova --- tests/time_tests/CMakeLists.txt | 3 +- tests/time_tests/README.md | 25 +++++++ tests/time_tests/common/CMakeLists.txt | 23 ------- tests/time_tests/common/cli.h | 57 ---------------- tests/time_tests/common/main.cpp | 52 --------------- tests/time_tests/common/statistics_writer.h | 55 ---------------- tests/time_tests/common/timer.h | 46 ------------- .../time_tests/ftti_pipeline/ftti_pipeline.h | 49 -------------- tests/time_tests/run_executable.py | 0 tests/time_tests/src/CMakeLists.txt | 19 ++++++ tests/time_tests/src/timetest_infer.cpp | 51 +++++++++++++++ .../time_tests/time-testhelper/CMakeLists.txt | 13 ++++ tests/time_tests/time-testhelper/cli.h | 65 +++++++++++++++++++ .../include/time-testhelper/timer.h | 32 +++++++++ tests/time_tests/time-testhelper/main.cpp | 55 ++++++++++++++++ .../time-testhelper/statistics_writer.h | 56 ++++++++++++++++ tests/time_tests/time-testhelper/timer.cpp | 29 +++++++++ 17 files changed, 347 insertions(+), 283 deletions(-) create mode 100644 tests/time_tests/README.md delete mode 100644 tests/time_tests/common/CMakeLists.txt delete mode 100644 tests/time_tests/common/cli.h delete mode 100644 tests/time_tests/common/main.cpp delete mode 100644 tests/time_tests/common/statistics_writer.h delete mode 100644 tests/time_tests/common/timer.h delete mode 100644 tests/time_tests/ftti_pipeline/ftti_pipeline.h mode change 100644 => 100755 tests/time_tests/run_executable.py create mode 100644 tests/time_tests/src/CMakeLists.txt create mode 100644 tests/time_tests/src/timetest_infer.cpp create mode 100644 tests/time_tests/time-testhelper/CMakeLists.txt create mode 100644 tests/time_tests/time-testhelper/cli.h create mode 100644 tests/time_tests/time-testhelper/include/time-testhelper/timer.h create mode 100644 tests/time_tests/time-testhelper/main.cpp create mode 100644 tests/time_tests/time-testhelper/statistics_writer.h create mode 100644 tests/time_tests/time-testhelper/timer.cpp diff --git a/tests/time_tests/CMakeLists.txt b/tests/time_tests/CMakeLists.txt index 24e66cfbe36..ee1f8d69b23 100644 --- a/tests/time_tests/CMakeLists.txt +++ b/tests/time_tests/CMakeLists.txt @@ -20,4 +20,5 @@ endif() find_package(InferenceEngineDeveloperPackage REQUIRED) -add_subdirectory(common) +add_subdirectory(time-testhelper) +add_subdirectory(src) diff --git a/tests/time_tests/README.md b/tests/time_tests/README.md new file mode 100644 index 00000000000..8ca98a23625 --- /dev/null +++ b/tests/time_tests/README.md @@ -0,0 +1,25 @@ +# Time Tests + +This test suite contains pipelines, which are executables. The pipelines measure +the time of their execution, both total and partial. A Python runner calls the +pipelines and calcuates the average execution time. + +## Prerequisites + +To build the time tests, you need to have the `build` folder, which is created +when you configure and build OpenVINO™. + +## Measure Time + +To build and run the tests, open a terminal and run the commands below: + +1. Build tests: +``` bash +cmake .. -DInferenceEngineDeveloperPackage_DIR=../../../build && make time-tests +``` + +2. Run test: +``` bash +./run_executable.py ../../../bin/intel64/Release/timetest_infer -m model.xml -d CPU +``` + diff --git a/tests/time_tests/common/CMakeLists.txt b/tests/time_tests/common/CMakeLists.txt deleted file mode 100644 index 86c0e302b84..00000000000 --- a/tests/time_tests/common/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) 2018-2019 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -set (TARGET_NAME "TimeTests") - -file (GLOB SRC - *.cpp - ../ftti_pipeline/*.cpp) - -file (GLOB HDR - *.h - ../ftti_pipeline/*.h) - -# Create library file from sources. -add_executable(${TARGET_NAME} ${HDR} ${SRC}) - -find_package(gflags REQUIRED) - -target_link_libraries(${TARGET_NAME} - gflags - ${InferenceEngine_LIBRARIES} - ) diff --git a/tests/time_tests/common/cli.h b/tests/time_tests/common/cli.h deleted file mode 100644 index b21758d80ee..00000000000 --- a/tests/time_tests/common/cli.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include - -/// @brief message for help argument -static const char help_message[] = "Print a usage message"; - -/// @brief message for model argument -static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with a trained compiled model."; - -/// @brief message for target device argument -static const char target_device_message[] = "Required. Specify a target device to infer on. " \ -"Use \"-d HETERO:\" format to specify HETERO plugin. " \ -"Use \"-d MULTI:\" format to specify MULTI plugin. " \ -"The application looks for a suitable plugin for the specified device."; - -/// @brief message for statistics path argument -static const char statistics_path_message[] = "Required. Path to a file to write statistics."; - -/// @brief Define flag for showing help message
-DEFINE_bool(h, false, help_message); - -/// @brief Declare flag for showing help message
-DECLARE_bool(help); - -/// @brief Define parameter for set model file
-/// It is a required parameter -DEFINE_string(m, "", model_message); - -/// @brief Define parameter for set target device to infer on
-/// It is a required parameter -DEFINE_string(d, "", target_device_message); - -/// @brief Define parameter for set path to a file to write statistics
-/// It is a required parameter -DEFINE_string(s, "", statistics_path_message); - -/** -* @brief This function show a help message -*/ -static void showUsage() { - std::cout << std::endl; - std::cout << "TimeTests [OPTION]" << std::endl; - std::cout << "Options:" << std::endl; - std::cout << std::endl; - std::cout << " -h, --help " << help_message << std::endl; - std::cout << " -m \"\" " << model_message << std::endl; - std::cout << " -d \"\" " << target_device_message << std::endl; - std::cout << " -s \"\" " << statistics_path_message << std::endl; -} diff --git a/tests/time_tests/common/main.cpp b/tests/time_tests/common/main.cpp deleted file mode 100644 index 164394f0909..00000000000 --- a/tests/time_tests/common/main.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "cli.h" -#include "statistics_writer.h" -#include "../ftti_pipeline/ftti_pipeline.h" - -#include - -/** -* @brief Parses command line and check required arguments -*/ -bool parseAndCheckCommandLine(int argc, char **argv) { - gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); - if (FLAGS_help || FLAGS_h) { - showUsage(); - return false; - } - - if (FLAGS_m.empty()) - throw std::logic_error("Model is required but not set. Please set -m option."); - - if (FLAGS_d.empty()) - throw std::logic_error("Device is required but not set. Please set -d option."); - - if (FLAGS_s.empty()) - throw std::logic_error("Statistics file path is required but not set. Please set -s option."); - - return true; -} - - -/** -* @brief Function calls `runPipeline` with mandatory time tracking of full run -*/ -int _runPipeline() { - SCOPED_TIMER(full_run); - return runPipeline(FLAGS_m, FLAGS_d); -} - - -/** -* @brief Main entry point -*/ -int main(int argc, char **argv) { - if (!parseAndCheckCommandLine(argc, argv)) - return -1; - - StatisticsWriter::Instance().setFile(FLAGS_s); - return _runPipeline(); -} \ No newline at end of file diff --git a/tests/time_tests/common/statistics_writer.h b/tests/time_tests/common/statistics_writer.h deleted file mode 100644 index f698ff16a13..00000000000 --- a/tests/time_tests/common/statistics_writer.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include - - -/** - * @brief Class response for writing provided statistics - * - * Object of the class is writing provided statistics to a specified - * file in YAML format. - */ -class StatisticsWriter { -private: - std::ofstream statistics_file; - - StatisticsWriter() = default; - StatisticsWriter(const StatisticsWriter&) = delete; - StatisticsWriter& operator=(const StatisticsWriter&) = delete; -public: - /** - * @brief Creates StatisticsWriter singleton object - */ - static StatisticsWriter& Instance(){ - static StatisticsWriter writer; - return writer; - } - - /** - * @brief Specifies, opens and validates statistics path for writing - */ - void setFile(const std::string &statistics_path) { - statistics_file.open(statistics_path); - if (!statistics_file.good()) { - std::stringstream err; - err << "Statistic file \"" << statistics_path << "\" can't be used for writing"; - throw std::runtime_error(err.str()); - } - } - - /** - * @brief Writes provided statistics in YAML format. - */ - void write(const std::pair &record) { - if (!statistics_file) - throw std::runtime_error("Statistic file path isn't set"); - statistics_file << record.first << ": " << record.second << "\n"; - } -}; diff --git a/tests/time_tests/common/timer.h b/tests/time_tests/common/timer.h deleted file mode 100644 index dc7a17abebf..00000000000 --- a/tests/time_tests/common/timer.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include - -#include "statistics_writer.h" - -using time_point = std::chrono::high_resolution_clock::time_point; - -/** -* @brief Class response for encapsulating time measurements. -* -* Object of a class measures time at start and finish of object's life cycle. -* When deleting, reports duration. -*/ -class Timer { -private: - std::string name; - time_point start_time; - -public: - /** - * @brief Constructs Timer object and measures start time - */ - Timer(const std::string &timer_name) { - name = timer_name; - start_time = std::chrono::high_resolution_clock::now(); - } - - /** - * @brief Destructs Timer object, measures duration and reports it - */ - ~Timer(){ - float duration = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - start_time).count(); - StatisticsWriter::Instance().write({name, duration}); - } -}; - -#define SCOPED_TIMER(timer_name) Timer timer_name(#timer_name); diff --git a/tests/time_tests/ftti_pipeline/ftti_pipeline.h b/tests/time_tests/ftti_pipeline/ftti_pipeline.h deleted file mode 100644 index f3fd289f9ba..00000000000 --- a/tests/time_tests/ftti_pipeline/ftti_pipeline.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "../common/timer.h" - -#include -using namespace InferenceEngine; - - -/** -* @brief Function that contain executable pipeline which will be called from main(). -* The function should not throw any exceptions and responsible for handling it by itself. -*/ -int runPipeline(const std::string &model, const std::string &device) { - auto pipeline = [](const std::string &model, const std::string &device){ - SCOPED_TIMER(first_time_to_inference); - - Core ie; - CNNNetwork cnnNetwork; - ExecutableNetwork exeNetwork; - - { - SCOPED_TIMER(read_network); - cnnNetwork = ie.ReadNetwork(model); - } - - { - SCOPED_TIMER(load_network); - ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device); - } - }; - - try { - pipeline(model, device); - } catch (const InferenceEngine::details::InferenceEngineException& iex) { - std::cerr << "Inference Engine pipeline failed with Inference Engine exception:\n" << iex.what(); - return 1; - } catch (const std::exception& ex) { - std::cerr << "Inference Engine pipeline failed with exception:\n" << ex.what(); - return 2; - } catch (...) { - std::cerr << "Inference Engine pipeline failed\n"; - return 3; - } - return 0; -} \ No newline at end of file diff --git a/tests/time_tests/run_executable.py b/tests/time_tests/run_executable.py old mode 100644 new mode 100755 diff --git a/tests/time_tests/src/CMakeLists.txt b/tests/time_tests/src/CMakeLists.txt new file mode 100644 index 00000000000..d04abf7141d --- /dev/null +++ b/tests/time_tests/src/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +# add dummy `time_tests` target combines all time tests +add_custom_target(time_tests) + +# Build test from every source file matchs *-pipeline.cpp. +# Test target name is source file name without extension. +FILE(GLOB tests "*-pipeline.cpp") + +foreach(test_source ${tests}) + get_filename_component(test_name ${test_source} NAME_WE) + add_executable(${test_name} ${test_source}) + + target_link_libraries(${test_name} PRIVATE IE::inference_engine time-testhelper) + + add_dependencies(time_tests ${test_name}) +endforeach() diff --git a/tests/time_tests/src/timetest_infer.cpp b/tests/time_tests/src/timetest_infer.cpp new file mode 100644 index 00000000000..59030eb8738 --- /dev/null +++ b/tests/time_tests/src/timetest_infer.cpp @@ -0,0 +1,51 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +#include "time-testhelper/timer.h" +using namespace InferenceEngine; + +/** + * @brief Function that contain executable pipeline which will be called from + * main(). The function should not throw any exceptions and responsible for + * handling it by itself. + */ +int runPipeline(const std::string &model, const std::string &device) { + auto pipeline = [](const std::string &model, const std::string &device) { + SCOPED_TIMER(first_time_to_inference); + + Core ie; + CNNNetwork cnnNetwork; + ExecutableNetwork exeNetwork; + + { + SCOPED_TIMER(read_network); + cnnNetwork = ie.ReadNetwork(model); + } + + { + SCOPED_TIMER(load_network); + ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device); + } + }; + + try { + pipeline(model, device); + } catch (const InferenceEngine::details::InferenceEngineException &iex) { + std::cerr + << "Inference Engine pipeline failed with Inference Engine exception:\n" + << iex.what(); + return 1; + } catch (const std::exception &ex) { + std::cerr << "Inference Engine pipeline failed with exception:\n" + << ex.what(); + return 2; + } catch (...) { + std::cerr << "Inference Engine pipeline failed\n"; + return 3; + } + return 0; +} \ No newline at end of file diff --git a/tests/time_tests/time-testhelper/CMakeLists.txt b/tests/time_tests/time-testhelper/CMakeLists.txt new file mode 100644 index 00000000000..6066dfdf00c --- /dev/null +++ b/tests/time_tests/time-testhelper/CMakeLists.txt @@ -0,0 +1,13 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set (TARGET_NAME "time-testhelper") + +find_package(gflags REQUIRED) + +file (GLOB SRC *.cpp) +add_library(${TARGET_NAME} STATIC ${SRC}) +target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") + +target_link_libraries(${TARGET_NAME} gflags) diff --git a/tests/time_tests/time-testhelper/cli.h b/tests/time_tests/time-testhelper/cli.h new file mode 100644 index 00000000000..f22e45b5b21 --- /dev/null +++ b/tests/time_tests/time-testhelper/cli.h @@ -0,0 +1,65 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +/// @brief message for help argument +static const char help_message[] = "Print a usage message"; + +/// @brief message for model argument +static const char model_message[] = + "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to " + "a .blob files with a trained compiled model."; + +/// @brief message for target device argument +static const char target_device_message[] = + "Required. Specify a target device to infer on. " + "Use \"-d HETERO:\" format to specify HETERO " + "plugin. " + "Use \"-d MULTI:\" format to specify MULTI " + "plugin. " + "The application looks for a suitable plugin for the specified device."; + +/// @brief message for statistics path argument +static const char statistics_path_message[] = + "Required. Path to a file to write statistics."; + +/// @brief Define flag for showing help message
+DEFINE_bool(h, false, help_message); + +/// @brief Declare flag for showing help message
+DECLARE_bool(help); + +/// @brief Define parameter for set model file
+/// It is a required parameter +DEFINE_string(m, "", model_message); + +/// @brief Define parameter for set target device to infer on
+/// It is a required parameter +DEFINE_string(d, "", target_device_message); + +/// @brief Define parameter for set path to a file to write statistics
+/// It is a required parameter +DEFINE_string(s, "", statistics_path_message); + +/** + * @brief This function show a help message + */ +static void showUsage() { + std::cout << std::endl; + std::cout << "TimeTests [OPTION]" << std::endl; + std::cout << "Options:" << std::endl; + std::cout << std::endl; + std::cout << " -h, --help " << help_message << std::endl; + std::cout << " -m \"\" " << model_message << std::endl; + std::cout << " -d \"\" " << target_device_message + << std::endl; + std::cout << " -s \"\" " << statistics_path_message + << std::endl; +} diff --git a/tests/time_tests/time-testhelper/include/time-testhelper/timer.h b/tests/time_tests/time-testhelper/include/time-testhelper/timer.h new file mode 100644 index 00000000000..9cec3ba123c --- /dev/null +++ b/tests/time_tests/time-testhelper/include/time-testhelper/timer.h @@ -0,0 +1,32 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace TimeTest { +using time_point = std::chrono::high_resolution_clock::time_point; + +/** Encapsulate time measurements. +Object of a class measures time at start and finish of object's life cycle. +When destroyed, reports duration. +*/ +class Timer { +private: + std::string name; + time_point start_time; + +public: + /// Constructs Timer object and measures start time. + Timer(const std::string &timer_name); + + /// Destructs Timer object, measures duration and reports it. + ~Timer(); +}; + +#define SCOPED_TIMER(timer_name) TimeTest::Timer timer_name(#timer_name); + +} // namespace TimeTest \ No newline at end of file diff --git a/tests/time_tests/time-testhelper/main.cpp b/tests/time_tests/time-testhelper/main.cpp new file mode 100644 index 00000000000..7b5af8c5278 --- /dev/null +++ b/tests/time_tests/time-testhelper/main.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "cli.h" +#include "statistics_writer.h" +#include "time-testhelper/timer.h" + +#include + +int runPipeline(const std::string &model, const std::string &device); + +/** + * @brief Parses command line and check required arguments + */ +bool parseAndCheckCommandLine(int argc, char **argv) { + gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); + if (FLAGS_help || FLAGS_h) { + showUsage(); + return false; + } + + if (FLAGS_m.empty()) + throw std::logic_error( + "Model is required but not set. Please set -m option."); + + if (FLAGS_d.empty()) + throw std::logic_error( + "Device is required but not set. Please set -d option."); + + if (FLAGS_s.empty()) + throw std::logic_error( + "Statistics file path is required but not set. Please set -s option."); + + return true; +} + +/** + * @brief Function calls `runPipeline` with mandatory time tracking of full run + */ +int _runPipeline() { + SCOPED_TIMER(full_run); + return runPipeline(FLAGS_m, FLAGS_d); +} + +/** + * @brief Main entry point + */ +int main(int argc, char **argv) { + if (!parseAndCheckCommandLine(argc, argv)) + return -1; + + StatisticsWriter::Instance().setFile(FLAGS_s); + return _runPipeline(); +} \ No newline at end of file diff --git a/tests/time_tests/time-testhelper/statistics_writer.h b/tests/time_tests/time-testhelper/statistics_writer.h new file mode 100644 index 00000000000..23a77fc7227 --- /dev/null +++ b/tests/time_tests/time-testhelper/statistics_writer.h @@ -0,0 +1,56 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +/** + * @brief Class response for writing provided statistics + * + * Object of the class is writing provided statistics to a specified + * file in YAML format. + */ +class StatisticsWriter { +private: + std::ofstream statistics_file; + + StatisticsWriter() = default; + StatisticsWriter(const StatisticsWriter &) = delete; + StatisticsWriter &operator=(const StatisticsWriter &) = delete; + +public: + /** + * @brief Creates StatisticsWriter singleton object + */ + static StatisticsWriter &Instance() { + static StatisticsWriter writer; + return writer; + } + + /** + * @brief Specifies, opens and validates statistics path for writing + */ + void setFile(const std::string &statistics_path) { + statistics_file.open(statistics_path); + if (!statistics_file.good()) { + std::stringstream err; + err << "Statistic file \"" << statistics_path + << "\" can't be used for writing"; + throw std::runtime_error(err.str()); + } + } + + /** + * @brief Writes provided statistics in YAML format. + */ + void write(const std::pair &record) { + if (!statistics_file) + throw std::runtime_error("Statistic file path isn't set"); + statistics_file << record.first << ": " << record.second << "\n"; + } +}; diff --git a/tests/time_tests/time-testhelper/timer.cpp b/tests/time_tests/time-testhelper/timer.cpp new file mode 100644 index 00000000000..44cc9d236ff --- /dev/null +++ b/tests/time_tests/time-testhelper/timer.cpp @@ -0,0 +1,29 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "time-testhelper/timer.h" +#include +#include +#include +#include + +#include "statistics_writer.h" + +using time_point = std::chrono::high_resolution_clock::time_point; + +namespace TimeTest { + +Timer::Timer(const std::string &timer_name) { + name = timer_name; + start_time = std::chrono::high_resolution_clock::now(); +} + +Timer::~Timer() { + float duration = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - start_time) + .count(); + StatisticsWriter::Instance().write({name, duration}); +} + +} // namespace TimeTest \ No newline at end of file