diff --git a/tests/lib/src/common_utils.cpp b/tests/lib/src/common_utils.cpp index a397866a440..3f445a2826f 100644 --- a/tests/lib/src/common_utils.cpp +++ b/tests/lib/src/common_utils.cpp @@ -85,4 +85,4 @@ void fillTensors(ov::InferRequest &infer_request, const std::vector +#include +#include + +#include "common_utils.h" +#include "memory_tests_helper/memory_counter.h" +#include "memory_tests_helper/utils.h" +#include "openvino/runtime/core.hpp" + + +/** + * @brief Function that contain executable pipeline which will be called from + * main(). The function should not throw any exceptions and responsible for + * handling it by itself. + */ +int runPipeline(const std::string &model, const std::string &device) { + auto pipeline = [](const std::string &model, const std::string &device) { + ov::Core ie; + std::shared_ptr network; + ov::CompiledModel compiled_model; + ov::InferRequest infer_request; + + ie.get_versions(device); + MEMORY_SNAPSHOT(load_plugin); + + if (MemoryTest::fileExt(model) == "blob") { + std::ifstream streamModel{model}; + compiled_model = ie.import_model(streamModel, device); + MEMORY_SNAPSHOT(import_network); + } else { + network = ie.read_model(model); + MEMORY_SNAPSHOT(read_network); + + compiled_model = ie.compile_model(network, device); + + MEMORY_SNAPSHOT(load_network); + } + MEMORY_SNAPSHOT(create_exenetwork); + + infer_request = compiled_model.create_infer_request(); + + auto inputs = network->inputs(); + fillTensors(infer_request, inputs); + MEMORY_SNAPSHOT(fill_inputs) + + infer_request.infer(); + MEMORY_SNAPSHOT(first_inference); + MEMORY_SNAPSHOT(full_run); + }; + + try { + pipeline(model, device); + } catch (const InferenceEngine::Exception &iex) { + std::cerr + << "Inference Engine pipeline failed with Inference Engine exception:\n" + << iex.what(); + return 1; + } catch (const std::exception &ex) { + std::cerr << "Inference Engine pipeline failed with exception:\n" + << ex.what(); + return 2; + } catch (...) { + std::cerr << "Inference Engine pipeline failed\n"; + return 3; + } + return 0; +} diff --git a/tests/memory_tests/src/memory_tests_helper/cli.h b/tests/memory_tests/src/memory_tests_helper/cli.h index 26aa149bbad..c6ed93db117 100644 --- a/tests/memory_tests/src/memory_tests_helper/cli.h +++ b/tests/memory_tests/src/memory_tests_helper/cli.h @@ -14,21 +14,21 @@ static const char help_message[] = "Print a usage message"; /// @brief message for model argument static const char model_message[] = - "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to " - "a .blob files with a trained compiled model."; + "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to " + "a .blob files with a trained compiled model."; /// @brief message for target device argument static const char target_device_message[] = - "Required. Specify a target device to infer on. " - "Use \"-d HETERO:\" format to specify HETERO " - "plugin. " - "Use \"-d MULTI:\" format to specify MULTI " - "plugin. " - "The application looks for a suitable plugin for the specified device."; + "Required. Specify a target device to infer on. " + "Use \"-d HETERO:\" format to specify HETERO " + "plugin. " + "Use \"-d MULTI:\" format to specify MULTI " + "plugin. " + "The application looks for a suitable plugin for the specified device."; /// @brief message for statistics path argument static const char statistics_path_message[] = - "Required. Path to a file to write statistics."; + "Required. Path to a file to write statistics."; /// @brief Define flag for showing help message
DEFINE_bool(h, false, help_message); @@ -52,14 +52,14 @@ DEFINE_string(s, "", statistics_path_message); * @brief This function show a help message */ static void showUsage() { - std::cout << std::endl; - std::cout << "TimeTests [OPTION]" << std::endl; - std::cout << "Options:" << std::endl; - std::cout << std::endl; - std::cout << " -h, --help " << help_message << std::endl; - std::cout << " -m \"\" " << model_message << std::endl; - std::cout << " -d \"\" " << target_device_message - << std::endl; - std::cout << " -s \"\" " << statistics_path_message - << std::endl; + std::cout << std::endl; + std::cout << "TimeTests [OPTION]" << std::endl; + std::cout << "Options:" << std::endl; + std::cout << std::endl; + std::cout << " -h, --help " << help_message << std::endl; + std::cout << " -m \"\" " << model_message << std::endl; + std::cout << " -d \"\" " << target_device_message + << std::endl; + std::cout << " -s \"\" " << statistics_path_message + << std::endl; } diff --git a/tests/memory_tests/src/memory_tests_helper/main.cpp b/tests/memory_tests/src/memory_tests_helper/main.cpp index 0d1802d103b..2d71137bfaa 100644 --- a/tests/memory_tests/src/memory_tests_helper/main.cpp +++ b/tests/memory_tests/src/memory_tests_helper/main.cpp @@ -14,25 +14,25 @@ int runPipeline(const std::string &model, const std::string &device); * @brief Parses command line and check required arguments */ bool parseAndCheckCommandLine(int argc, char **argv) { - gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); - if (FLAGS_help || FLAGS_h) { - showUsage(); - return false; - } + gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); + if (FLAGS_help || FLAGS_h) { + showUsage(); + return false; + } - if (FLAGS_m.empty()) - throw std::logic_error( - "Model is required but not set. Please set -m option."); + if (FLAGS_m.empty()) + throw std::logic_error( + "Model is required but not set. Please set -m option."); - if (FLAGS_d.empty()) - throw std::logic_error( - "Device is required but not set. Please set -d option."); + if (FLAGS_d.empty()) + throw std::logic_error( + "Device is required but not set. Please set -d option."); - if (FLAGS_s.empty()) - throw std::logic_error( - "Statistics file path is required but not set. Please set -s option."); + if (FLAGS_s.empty()) + throw std::logic_error( + "Statistics file path is required but not set. Please set -s option."); - return true; + return true; } /** @@ -48,11 +48,11 @@ int _runPipeline() { * @brief Main entry point */ int main(int argc, char **argv) { - if (!parseAndCheckCommandLine(argc, argv)) - return -1; + if (!parseAndCheckCommandLine(argc, argv)) + return -1; - auto status = _runPipeline(); - StatisticsWriter::Instance().setFile(FLAGS_s); - StatisticsWriter::Instance().write(); - return status; + auto status = _runPipeline(); + StatisticsWriter::Instance().setFile(FLAGS_s); + StatisticsWriter::Instance().write(); + return status; } diff --git a/tests/memory_tests/src/memory_tests_helper/memory_counter.cpp b/tests/memory_tests/src/memory_tests_helper/memory_counter.cpp index 8a379249453..ed5fd218e27 100644 --- a/tests/memory_tests/src/memory_tests_helper/memory_counter.cpp +++ b/tests/memory_tests/src/memory_tests_helper/memory_counter.cpp @@ -14,105 +14,111 @@ #include #include #else + #include #include + #endif #include "statistics_writer.h" namespace MemoryTest { #ifdef _WIN32 -static PROCESS_MEMORY_COUNTERS getMemoryInfo() { - static PROCESS_MEMORY_COUNTERS pmc; - pmc.cb = sizeof(PROCESS_MEMORY_COUNTERS); - GetProcessMemoryInfo(GetCurrentProcess(),&pmc, pmc.cb); - return pmc; -} - -size_t getVmSizeInKB() { - return getMemoryInfo().PagefileUsage / 1024; + static PROCESS_MEMORY_COUNTERS getMemoryInfo() { + static PROCESS_MEMORY_COUNTERS pmc; + pmc.cb = sizeof(PROCESS_MEMORY_COUNTERS); + GetProcessMemoryInfo(GetCurrentProcess(),&pmc, pmc.cb); + return pmc; } -size_t getVmPeakInKB() { - return getMemoryInfo().PeakPagefileUsage / 1024; - } + size_t getVmSizeInKB() { + return getMemoryInfo().PagefileUsage / 1024; + } -size_t getVmRSSInKB() { - return getMemoryInfo().WorkingSetSize / 1024; - } + size_t getVmPeakInKB() { + return getMemoryInfo().PeakPagefileUsage / 1024; + } -size_t getVmHWMInKB() { - return getMemoryInfo().PeakWorkingSetSize / 1024; - } + size_t getVmRSSInKB() { + return getMemoryInfo().WorkingSetSize / 1024; + } -size_t getThreadsNum() { - // first determine the id of the current process - DWORD const id = GetCurrentProcessId(); + size_t getVmHWMInKB() { + return getMemoryInfo().PeakWorkingSetSize / 1024; + } - // then get a process list snapshot. - HANDLE const snapshot = CreateToolhelp32Snapshot( TH32CS_SNAPALL, 0 ); + size_t getThreadsNum() { + // first determine the id of the current process + DWORD const id = GetCurrentProcessId(); - // initialize the process entry structure. - PROCESSENTRY32 entry = { 0 }; - entry.dwSize = sizeof( entry ); + // then get a process list snapshot. + HANDLE const snapshot = CreateToolhelp32Snapshot( TH32CS_SNAPALL, 0 ); - // get the first process info. - BOOL ret = true; - ret = Process32First( snapshot, &entry ); - while( ret && entry.th32ProcessID != id ) { - ret = Process32Next( snapshot, &entry ); - } - CloseHandle( snapshot ); - return ret - ? entry.cntThreads - : -1; - } + // initialize the process entry structure. + PROCESSENTRY32 entry = { 0 }; + entry.dwSize = sizeof( entry ); + + // get the first process info. + BOOL ret = true; + ret = Process32First( snapshot, &entry ); + while( ret && entry.th32ProcessID != id ) { + ret = Process32Next( snapshot, &entry ); + } + CloseHandle( snapshot ); + return ret + ? entry.cntThreads + : -1; + } #else /// Parses number from provided string -static int parseLine(std::string line) { - std::string res = ""; - for (auto c: line) - if (isdigit(c)) - res += c; - if (res.empty()) - // If number wasn't found return -1 - return -1; - return std::stoi(res); -} - -size_t getSystemDataByName(char *name){ - FILE* file = fopen("/proc/self/status", "r"); - size_t result = 0; - if (file != nullptr) { - char line[128]; - - while (fgets(line, 128, file) != NULL) { - if (strncmp(line, name, strlen(name)) == 0) { - result = parseLine(line); - break; - } - } - fclose(file); + static int parseLine(std::string line) { + std::string res = ""; + for (auto c: line) + if (isdigit(c)) + res += c; + if (res.empty()) + // If number wasn't found return -1 + return -1; + return std::stoi(res); } - return result; -} -size_t getVmSizeInKB() {return getSystemDataByName((char*) "VmSize:");} -size_t getVmPeakInKB() {return getSystemDataByName((char*) "VmPeak:");} -size_t getVmRSSInKB() {return getSystemDataByName((char*) "VmRSS:");} -size_t getVmHWMInKB() {return getSystemDataByName((char*) "VmHWM:");} -size_t getThreadsNum() {return getSystemDataByName((char*) "Threads:");} + size_t getSystemDataByName(char *name) { + FILE *file = fopen("/proc/self/status", "r"); + size_t result = 0; + if (file != nullptr) { + char line[128]; + + while (fgets(line, 128, file) != NULL) { + if (strncmp(line, name, strlen(name)) == 0) { + result = parseLine(line); + break; + } + } + fclose(file); + } + return result; + } + + size_t getVmSizeInKB() { return getSystemDataByName((char *) "VmSize:"); } + + size_t getVmPeakInKB() { return getSystemDataByName((char *) "VmPeak:"); } + + size_t getVmRSSInKB() { return getSystemDataByName((char *) "VmRSS:"); } + + size_t getVmHWMInKB() { return getSystemDataByName((char *) "VmHWM:"); } + + size_t getThreadsNum() { return getSystemDataByName((char *) "Threads:"); } #endif -MemoryCounter::MemoryCounter(const std::string &mem_counter_name) { - name = mem_counter_name; - std::vector memory_measurements = {getVmRSSInKB(), getVmHWMInKB(), getVmSizeInKB(), - getVmPeakInKB(), getThreadsNum()}; - StatisticsWriter::Instance().addMemCounterToStructure({name, memory_measurements}); -} + MemoryCounter::MemoryCounter(const std::string &mem_counter_name) { + name = mem_counter_name; + std::vector memory_measurements = {getVmRSSInKB(), getVmHWMInKB(), getVmSizeInKB(), + getVmPeakInKB(), getThreadsNum()}; + StatisticsWriter::Instance().addMemCounterToStructure({name, memory_measurements}); + } } // namespace MemoryTest diff --git a/tests/memory_tests/src/memory_tests_helper/statistics_writer.h b/tests/memory_tests/src/memory_tests_helper/statistics_writer.h index b07c9c5c605..535536b0dde 100644 --- a/tests/memory_tests/src/memory_tests_helper/statistics_writer.h +++ b/tests/memory_tests/src/memory_tests_helper/statistics_writer.h @@ -22,59 +22,61 @@ */ class StatisticsWriter { private: - std::ofstream statistics_file; + std::ofstream statistics_file; - std::map> mem_structure; // mem_counter_name, memory measurements - std::vector mem_struct_order; + std::map> mem_structure; // mem_counter_name, memory measurements + std::vector mem_struct_order; - StatisticsWriter() = default; - StatisticsWriter(const StatisticsWriter &) = delete; - StatisticsWriter &operator=(const StatisticsWriter &) = delete; + StatisticsWriter() = default; + + StatisticsWriter(const StatisticsWriter &) = delete; + + StatisticsWriter &operator=(const StatisticsWriter &) = delete; public: - /** - * @brief Creates StatisticsWriter singleton object - */ - static StatisticsWriter &Instance() { - static StatisticsWriter writer; - return writer; - } - - /** - * @brief Specifies, opens and validates statistics path for writing - */ - void setFile(const std::string &statistics_path) { - statistics_file.open(statistics_path); - if (!statistics_file.good()) { - std::stringstream err; - err << "Statistic file \"" << statistics_path - << "\" can't be used for writing"; - throw std::runtime_error(err.str()); + /** + * @brief Creates StatisticsWriter singleton object + */ + static StatisticsWriter &Instance() { + static StatisticsWriter writer; + return writer; } - } - /** - * @brief Creates counter structure - */ - void addMemCounterToStructure(const std::pair> &record) { - mem_struct_order.push_back(record.first); - mem_structure[record.first] = record.second; - } - - /** - * @brief Writes provided statistics in YAML format. - */ - void write() { - if (!statistics_file) - throw std::runtime_error("Statistic file path isn't set"); - for (auto& mem_counter: mem_struct_order) { - statistics_file << mem_counter << ":" << '\n' - << SEPARATOR << "vmrss: " << mem_structure[mem_counter][0] << '\n' - << SEPARATOR << "vmhwm: " << mem_structure[mem_counter][1] << '\n' - << SEPARATOR << "vmsize: " << mem_structure[mem_counter][2] << '\n' - << SEPARATOR << "vmpeak: " << mem_structure[mem_counter][3] << '\n' - << SEPARATOR << "threads: " << mem_structure[mem_counter][4] << '\n'; + /** + * @brief Specifies, opens and validates statistics path for writing + */ + void setFile(const std::string &statistics_path) { + statistics_file.open(statistics_path); + if (!statistics_file.good()) { + std::stringstream err; + err << "Statistic file \"" << statistics_path + << "\" can't be used for writing"; + throw std::runtime_error(err.str()); + } + } + + /** + * @brief Creates counter structure + */ + void addMemCounterToStructure(const std::pair> &record) { + mem_struct_order.push_back(record.first); + mem_structure[record.first] = record.second; + } + + /** + * @brief Writes provided statistics in YAML format. + */ + void write() { + if (!statistics_file) + throw std::runtime_error("Statistic file path isn't set"); + for (auto &mem_counter: mem_struct_order) { + statistics_file << mem_counter << ":" << '\n' + << SEPARATOR << "vmrss: " << mem_structure[mem_counter][0] << '\n' + << SEPARATOR << "vmhwm: " << mem_structure[mem_counter][1] << '\n' + << SEPARATOR << "vmsize: " << mem_structure[mem_counter][2] << '\n' + << SEPARATOR << "vmpeak: " << mem_structure[mem_counter][3] << '\n' + << SEPARATOR << "threads: " << mem_structure[mem_counter][4] << '\n'; + } + statistics_file << "---" << '\n' << "measurement_unit: Kb"; } - statistics_file << "---" << '\n' << "measurement_unit: Kb"; - } }; diff --git a/tests/memory_tests/test_runner/conftest.py b/tests/memory_tests/test_runner/conftest.py index 610faf8adda..2971fb972db 100644 --- a/tests/memory_tests/test_runner/conftest.py +++ b/tests/memory_tests/test_runner/conftest.py @@ -18,17 +18,16 @@ This plugin adds the following command-line options: import hashlib import json import logging -import tempfile # pylint:disable=import-error import os +import pytest import sys +import tempfile +import yaml from copy import deepcopy from inspect import getsourcefile -from pathlib import Path - -import pytest -import yaml from jsonschema import validate, ValidationError +from pathlib import Path UTILS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "utils") sys.path.insert(0, str(UTILS_DIR)) @@ -43,7 +42,6 @@ sys.path.append(MEMORY_TESTS_DIR) from test_runner.utils import query_memory_timeline, REFS_FACTOR - OMZ_NUM_ATTEMPTS = 6 @@ -168,6 +166,8 @@ def executable(request): def niter(request): """Fixture function for command-line option.""" return request.config.getoption('niter') + + # -------------------- CLI options -------------------- diff --git a/tests/memory_tests/test_runner/utils.py b/tests/memory_tests/test_runner/utils.py index 4f2cf879b06..8a0ae4b408c 100644 --- a/tests/memory_tests/test_runner/utils.py +++ b/tests/memory_tests/test_runner/utils.py @@ -37,6 +37,7 @@ def _transpose_dicts(items, template=None): def query_memory_timeline(records, db_url, db_name, db_collection, max_items=20, similarity=TIMELINE_SIMILARITY): """ Query database for similar memory items committed previously """ + def timeline_key(item): """ Defines order for timeline report entries """ @@ -45,10 +46,10 @@ def query_memory_timeline(records, db_url, db_name, db_collection, max_items=20, if len(item['results'][step_name]['vmhwm']) <= 1: return 1 order = item['results'][step_name]['vmhwm']["avg"][-1] - item['results'][step_name]['vmhwm']["avg"][-2] + \ - item['results'][step_name]['vmrss']["avg"][-1] - item['results'][step_name]['vmrss']["avg"][-2] + item['results'][step_name]['vmrss']["avg"][-1] - item['results'][step_name]['vmrss']["avg"][-2] if not item['status']: # ensure failed cases are always on top - order += sys.maxsize/2 + order += sys.maxsize / 2 return order client = MongoClient(db_url)