Split time-tests common library (#2173)

Split time-tests common library

Add a README.md with workflow description.
Defined "timetest_" suffix for all time tests.
Applied clang-format-9 and added a README.md

Co-authored-by: Alina Alborova <alina.alborova@intel.com>

tests/time_tests/CMakeLists.txt

@@ -20,4 +20,5 @@ endif()
 
 find_package(InferenceEngineDeveloperPackage REQUIRED)
 
-add_subdirectory(common)
+add_subdirectory(time-testhelper)
+add_subdirectory(src)

tests/time_tests/README.md

@@ -0,0 +1,25 @@
+# Time Tests
+
+This test suite contains pipelines, which are executables. The pipelines measure
+the time of their execution, both total and partial. A Python runner calls the
+pipelines and calcuates the average execution time.
+
+## Prerequisites
+
+To build the time tests, you need to have the `build` folder, which is created
+when you configure and build OpenVINO™.
+
+## Measure Time
+
+To build and run the tests, open a terminal and run the commands below:
+
+1. Build tests:
+``` bash
+cmake .. -DInferenceEngineDeveloperPackage_DIR=../../../build && make time-tests
+```
+
+2. Run test:
+``` bash
+./run_executable.py ../../../bin/intel64/Release/timetest_infer -m model.xml -d CPU
+```
+

tests/time_tests/common/CMakeLists.txt

@@ -1,23 +0,0 @@
-# Copyright (C) 2018-2019 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set (TARGET_NAME "TimeTests")
-
-file (GLOB SRC
-        *.cpp
-        ../ftti_pipeline/*.cpp)
-
-file (GLOB HDR
-        *.h
-        ../ftti_pipeline/*.h)
-
-# Create library file from sources.
-add_executable(${TARGET_NAME} ${HDR} ${SRC})
-
-find_package(gflags REQUIRED)
-
-target_link_libraries(${TARGET_NAME}
-        gflags
-        ${InferenceEngine_LIBRARIES}
-        )

tests/time_tests/common/cli.h

@@ -1,57 +0,0 @@
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#pragma once
-
-#include <string>
-#include <vector>
-#include <gflags/gflags.h>
-#include <iostream>
-
-/// @brief message for help argument
-static const char help_message[] = "Print a usage message";
-
-/// @brief message for model argument
-static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with a trained compiled model.";
-
-/// @brief message for target device argument
-static const char target_device_message[] = "Required. Specify a target device to infer on. " \
-"Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO plugin. " \
-"Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI plugin. " \
-"The application looks for a suitable plugin for the specified device.";
-
-/// @brief message for statistics path argument
-static const char statistics_path_message[] = "Required. Path to a file to write statistics.";
-
-/// @brief Define flag for showing help message <br>
-DEFINE_bool(h, false, help_message);
-
-/// @brief Declare flag for showing help message <br>
-DECLARE_bool(help);
-
-/// @brief Define parameter for set model file <br>
-/// It is a required parameter
-DEFINE_string(m, "", model_message);
-
-/// @brief Define parameter for set target device to infer on <br>
-/// It is a required parameter
-DEFINE_string(d, "", target_device_message);
-
-/// @brief Define parameter for set path to a file to write statistics <br>
-/// It is a required parameter
-DEFINE_string(s, "", statistics_path_message);
-
-/**
-* @brief This function show a help message
-*/
-static void showUsage() {
-    std::cout << std::endl;
-    std::cout << "TimeTests [OPTION]" << std::endl;
-    std::cout << "Options:" << std::endl;
-    std::cout << std::endl;
-    std::cout << "    -h, --help                " << help_message << std::endl;
-    std::cout << "    -m \"<path>\"               " << model_message << std::endl;
-    std::cout << "    -d \"<device>\"             " << target_device_message << std::endl;
-    std::cout << "    -s \"<path>\"               " << statistics_path_message << std::endl;
-}

tests/time_tests/common/main.cpp

@@ -1,52 +0,0 @@
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#include "cli.h"
-#include "statistics_writer.h"
-#include "../ftti_pipeline/ftti_pipeline.h"
-
-#include <iostream>
-
-/**
-* @brief Parses command line and check required arguments
-*/
-bool parseAndCheckCommandLine(int argc, char **argv) {
-    gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
-    if (FLAGS_help || FLAGS_h) {
-        showUsage();
-        return false;
-    }
-
-    if (FLAGS_m.empty())
-        throw std::logic_error("Model is required but not set. Please set -m option.");
-
-    if (FLAGS_d.empty())
-        throw std::logic_error("Device is required but not set. Please set -d option.");
-
-    if (FLAGS_s.empty())
-        throw std::logic_error("Statistics file path is required but not set. Please set -s option.");
-
-    return true;
-}
-
-
-/**
-* @brief Function calls `runPipeline` with mandatory time tracking of full run
-*/
-int _runPipeline() {
-    SCOPED_TIMER(full_run);
-    return runPipeline(FLAGS_m, FLAGS_d);
-}
-
-
-/**
-* @brief Main entry point
-*/
-int main(int argc, char **argv) {
-    if (!parseAndCheckCommandLine(argc, argv))
-        return -1;
-
-    StatisticsWriter::Instance().setFile(FLAGS_s);
-    return _runPipeline();
-}

tests/time_tests/common/statistics_writer.h

@@ -1,55 +0,0 @@
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#pragma once
-
-#include <string>
-#include <sstream>
-#include <fstream>
-#include <cstdio>
-
-
-/**
- * @brief Class response for writing provided statistics
- *
- * Object of the class is writing provided statistics to a specified
- * file in YAML format.
- */
-class StatisticsWriter {
-private:
-    std::ofstream statistics_file;
-
-    StatisticsWriter() = default;
-    StatisticsWriter(const StatisticsWriter&) = delete;
-    StatisticsWriter& operator=(const StatisticsWriter&) = delete;
-public:
-    /**
-     * @brief Creates StatisticsWriter singleton object
-     */
-    static StatisticsWriter& Instance(){
-        static StatisticsWriter writer;
-        return writer;
-    }
-
-    /**
-     * @brief Specifies, opens and validates statistics path for writing
-     */
-    void setFile(const std::string &statistics_path) {
-        statistics_file.open(statistics_path);
-        if (!statistics_file.good()) {
-            std::stringstream err;
-            err << "Statistic file \"" << statistics_path << "\" can't be used for writing";
-            throw std::runtime_error(err.str());
-        }
-    }
-
-    /**
-     * @brief Writes provided statistics in YAML format.
-     */
-    void write(const std::pair<std::string, float> &record) {
-        if (!statistics_file)
-            throw std::runtime_error("Statistic file path isn't set");
-        statistics_file << record.first << ": " << record.second << "\n";
-    }
-};

tests/time_tests/common/timer.h

@@ -1,46 +0,0 @@
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#pragma once
-
-#include <string>
-#include <chrono>
-#include <fstream>
-#include <memory>
-
-#include "statistics_writer.h"
-
-using time_point = std::chrono::high_resolution_clock::time_point;
-
-/**
-* @brief Class response for encapsulating time measurements.
-*
-* Object of a class measures time at start and finish of object's life cycle.
-* When deleting, reports duration.
-*/
-class Timer {
-private:
-    std::string name;
-    time_point start_time;
-
-public:
-    /**
-     * @brief Constructs Timer object and measures start time
-     */
-    Timer(const std::string &timer_name) {
-        name = timer_name;
-        start_time = std::chrono::high_resolution_clock::now();
-    }
-
-    /**
-     * @brief Destructs Timer object, measures duration and reports it
-     */
-    ~Timer(){
-        float duration = std::chrono::duration_cast<std::chrono::microseconds>(
-                std::chrono::high_resolution_clock::now() - start_time).count();
-        StatisticsWriter::Instance().write({name, duration});
-    }
-};
-
-#define SCOPED_TIMER(timer_name) Timer timer_name(#timer_name);

tests/time_tests/ftti_pipeline/ftti_pipeline.h

@@ -1,49 +0,0 @@
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#pragma once
-
-#include "../common/timer.h"
-
-#include <inference_engine.hpp>
-using namespace InferenceEngine;
-
-
-/**
-* @brief Function that contain executable pipeline which will be called from main().
-* The function should not throw any exceptions and responsible for handling it by itself.
-*/
-int runPipeline(const std::string &model, const std::string &device) {
-    auto pipeline = [](const std::string &model, const std::string &device){
-        SCOPED_TIMER(first_time_to_inference);
-
-        Core ie;
-        CNNNetwork cnnNetwork;
-        ExecutableNetwork exeNetwork;
-
-        {
-            SCOPED_TIMER(read_network);
-            cnnNetwork = ie.ReadNetwork(model);
-        }
-
-        {
-            SCOPED_TIMER(load_network);
-            ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
-        }
-    };
-
-    try {
-        pipeline(model, device);
-    } catch (const InferenceEngine::details::InferenceEngineException& iex) {
-        std::cerr << "Inference Engine pipeline failed with Inference Engine exception:\n" << iex.what();
-        return 1;
-    } catch (const std::exception& ex) {
-        std::cerr << "Inference Engine pipeline failed with exception:\n" << ex.what();
-        return 2;
-    } catch (...) {
-        std::cerr << "Inference Engine pipeline failed\n";
-        return 3;
-    }
-    return 0;
-}

tests/time_tests/src/CMakeLists.txt

@@ -0,0 +1,19 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# add dummy `time_tests` target combines all time tests
+add_custom_target(time_tests)
+
+# Build test from every source file matchs *-pipeline.cpp.
+# Test target name is source file name without extension.
+FILE(GLOB tests "*-pipeline.cpp")
+
+foreach(test_source ${tests})
+    get_filename_component(test_name ${test_source} NAME_WE)
+    add_executable(${test_name} ${test_source})
+
+    target_link_libraries(${test_name} PRIVATE IE::inference_engine time-testhelper)
+
+    add_dependencies(time_tests ${test_name})
+endforeach()

tests/time_tests/src/timetest_infer.cpp

@@ -0,0 +1,51 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <inference_engine.hpp>
+#include <iostream>
+
+#include "time-testhelper/timer.h"
+using namespace InferenceEngine;
+
+/**
+ * @brief Function that contain executable pipeline which will be called from
+ * main(). The function should not throw any exceptions and responsible for
+ * handling it by itself.
+ */
+int runPipeline(const std::string &model, const std::string &device) {
+  auto pipeline = [](const std::string &model, const std::string &device) {
+    SCOPED_TIMER(first_time_to_inference);
+
+    Core ie;
+    CNNNetwork cnnNetwork;
+    ExecutableNetwork exeNetwork;
+
+    {
+      SCOPED_TIMER(read_network);
+      cnnNetwork = ie.ReadNetwork(model);
+    }
+
+    {
+      SCOPED_TIMER(load_network);
+      ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
+    }
+  };
+
+  try {
+    pipeline(model, device);
+  } catch (const InferenceEngine::details::InferenceEngineException &iex) {
+    std::cerr
+        << "Inference Engine pipeline failed with Inference Engine exception:\n"
+        << iex.what();
+    return 1;
+  } catch (const std::exception &ex) {
+    std::cerr << "Inference Engine pipeline failed with exception:\n"
+              << ex.what();
+    return 2;
+  } catch (...) {
+    std::cerr << "Inference Engine pipeline failed\n";
+    return 3;
+  }
+  return 0;
+}

tests/time_tests/time-testhelper/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set (TARGET_NAME "time-testhelper")
+
+find_package(gflags REQUIRED)
+
+file (GLOB SRC *.cpp)
+add_library(${TARGET_NAME} STATIC ${SRC})
+target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
+
+target_link_libraries(${TARGET_NAME} gflags)

tests/time_tests/time-testhelper/cli.h

@@ -0,0 +1,65 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <gflags/gflags.h>
+#include <iostream>
+#include <string>
+#include <vector>
+
+/// @brief message for help argument
+static const char help_message[] = "Print a usage message";
+
+/// @brief message for model argument
+static const char model_message[] =
+    "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to "
+    "a .blob files with a trained compiled model.";
+
+/// @brief message for target device argument
+static const char target_device_message[] =
+    "Required. Specify a target device to infer on. "
+    "Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO "
+    "plugin. "
+    "Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI "
+    "plugin. "
+    "The application looks for a suitable plugin for the specified device.";
+
+/// @brief message for statistics path argument
+static const char statistics_path_message[] =
+    "Required. Path to a file to write statistics.";
+
+/// @brief Define flag for showing help message <br>
+DEFINE_bool(h, false, help_message);
+
+/// @brief Declare flag for showing help message <br>
+DECLARE_bool(help);
+
+/// @brief Define parameter for set model file <br>
+/// It is a required parameter
+DEFINE_string(m, "", model_message);
+
+/// @brief Define parameter for set target device to infer on <br>
+/// It is a required parameter
+DEFINE_string(d, "", target_device_message);
+
+/// @brief Define parameter for set path to a file to write statistics <br>
+/// It is a required parameter
+DEFINE_string(s, "", statistics_path_message);
+
+/**
+ * @brief This function show a help message
+ */
+static void showUsage() {
+  std::cout << std::endl;
+  std::cout << "TimeTests [OPTION]" << std::endl;
+  std::cout << "Options:" << std::endl;
+  std::cout << std::endl;
+  std::cout << "    -h, --help                " << help_message << std::endl;
+  std::cout << "    -m \"<path>\"               " << model_message << std::endl;
+  std::cout << "    -d \"<device>\"             " << target_device_message
+            << std::endl;
+  std::cout << "    -s \"<path>\"               " << statistics_path_message
+            << std::endl;
+}

tests/time_tests/time-testhelper/include/time-testhelper/timer.h

@@ -0,0 +1,32 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <chrono>
+#include <string>
+
+namespace TimeTest {
+using time_point = std::chrono::high_resolution_clock::time_point;
+
+/** Encapsulate time measurements.
+Object of a class measures time at start and finish of object's life cycle.
+When destroyed, reports duration.
+*/
+class Timer {
+private:
+  std::string name;
+  time_point start_time;
+
+public:
+  /// Constructs Timer object and measures start time.
+  Timer(const std::string &timer_name);
+
+  /// Destructs Timer object, measures duration and reports it.
+  ~Timer();
+};
+
+#define SCOPED_TIMER(timer_name) TimeTest::Timer timer_name(#timer_name);
+
+} // namespace TimeTest

tests/time_tests/time-testhelper/main.cpp

@@ -0,0 +1,55 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "cli.h"
+#include "statistics_writer.h"
+#include "time-testhelper/timer.h"
+
+#include <iostream>
+
+int runPipeline(const std::string &model, const std::string &device);
+
+/**
+ * @brief Parses command line and check required arguments
+ */
+bool parseAndCheckCommandLine(int argc, char **argv) {
+  gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
+  if (FLAGS_help || FLAGS_h) {
+    showUsage();
+    return false;
+  }
+
+  if (FLAGS_m.empty())
+    throw std::logic_error(
+        "Model is required but not set. Please set -m option.");
+
+  if (FLAGS_d.empty())
+    throw std::logic_error(
+        "Device is required but not set. Please set -d option.");
+
+  if (FLAGS_s.empty())
+    throw std::logic_error(
+        "Statistics file path is required but not set. Please set -s option.");
+
+  return true;
+}
+
+/**
+ * @brief Function calls `runPipeline` with mandatory time tracking of full run
+ */
+int _runPipeline() {
+  SCOPED_TIMER(full_run);
+  return runPipeline(FLAGS_m, FLAGS_d);
+}
+
+/**
+ * @brief Main entry point
+ */
+int main(int argc, char **argv) {
+  if (!parseAndCheckCommandLine(argc, argv))
+    return -1;
+
+  StatisticsWriter::Instance().setFile(FLAGS_s);
+  return _runPipeline();
+}

tests/time_tests/time-testhelper/statistics_writer.h

@@ -0,0 +1,56 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <cstdio>
+#include <fstream>
+#include <sstream>
+#include <string>
+
+/**
+ * @brief Class response for writing provided statistics
+ *
+ * Object of the class is writing provided statistics to a specified
+ * file in YAML format.
+ */
+class StatisticsWriter {
+private:
+  std::ofstream statistics_file;
+
+  StatisticsWriter() = default;
+  StatisticsWriter(const StatisticsWriter &) = delete;
+  StatisticsWriter &operator=(const StatisticsWriter &) = delete;
+
+public:
+  /**
+   * @brief Creates StatisticsWriter singleton object
+   */
+  static StatisticsWriter &Instance() {
+    static StatisticsWriter writer;
+    return writer;
+  }
+
+  /**
+   * @brief Specifies, opens and validates statistics path for writing
+   */
+  void setFile(const std::string &statistics_path) {
+    statistics_file.open(statistics_path);
+    if (!statistics_file.good()) {
+      std::stringstream err;
+      err << "Statistic file \"" << statistics_path
+          << "\" can't be used for writing";
+      throw std::runtime_error(err.str());
+    }
+  }
+
+  /**
+   * @brief Writes provided statistics in YAML format.
+   */
+  void write(const std::pair<std::string, float> &record) {
+    if (!statistics_file)
+      throw std::runtime_error("Statistic file path isn't set");
+    statistics_file << record.first << ": " << record.second << "\n";
+  }
+};

tests/time_tests/time-testhelper/timer.cpp

@@ -0,0 +1,29 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "time-testhelper/timer.h"
+#include <chrono>
+#include <fstream>
+#include <memory>
+#include <string>
+
+#include "statistics_writer.h"
+
+using time_point = std::chrono::high_resolution_clock::time_point;
+
+namespace TimeTest {
+
+Timer::Timer(const std::string &timer_name) {
+  name = timer_name;
+  start_time = std::chrono::high_resolution_clock::now();
+}
+
+Timer::~Timer() {
+  float duration = std::chrono::duration_cast<std::chrono::microseconds>(
+                       std::chrono::high_resolution_clock::now() - start_time)
+                       .count();
+  StatisticsWriter::Instance().write({name, duration});
+}
+
+} // namespace TimeTest