IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Stress] Redesign of MemCheckTests (#650)

Browse Source 
* [Stress] Redesigned MemCheckTests: 1. Added MemCheckPipeline to incapsulate measures and logging. 2. Moved references to array

* [Stress] Added tracking of THREADS in MemCheckTests
This commit is contained in:
Vitaliy Urusovskij 2020-07-23 11:07:13 +03:00 committed by GitHub
parent dce1063526
commit 6fcad95c71
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 203 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
tests/stress_tests/memcheck_tests/tests.cpp
View File

@ -9,49 +9,87 @@
#include <gtest/gtest.h>
#define checkRefVmValues() \
if (!Environment::Instance().getCollectResultsOnly()) { \
ASSERT_GT(test_refs.ref_vmsize, 0) << "Reference value of VmSize is less than 0. Value: " \
<< test_refs.ref_vmsize; \
ASSERT_GT(test_refs.ref_vmsize, 0) << "Reference value of VmPeak is less than 0. Value: " \
<< test_refs.ref_vmpeak; \
ASSERT_GT(test_refs.ref_vmrss, 0) << "Reference value of VmRSS is less than 0. Value: " \
<< test_refs.ref_vmrss; \
ASSERT_GT(test_refs.ref_vmrss, 0) << "Reference value of VmHWM is less than 0. Value: " \
<< test_refs.ref_vmhwm; \
}
#include <inference_engine.hpp>
using namespace InferenceEngine;
class MemCheckTestSuite : public ::testing::TestWithParam<TestCase> {
public:
std::string test_name, model, model_name, device;
TestReferences test_refs;
void SetUp() override {
const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
test_name = std::string(test_info->name()).substr(0, std::string(test_info->name()).find('/'));
//const std::string full_test_name = std::string(test_info->test_case_name()) + "." + std::string(test_info->name());
const auto& test_params = GetParam();
model = test_params.model;
model_name = test_params.model_name;
device = test_params.device;
test_refs.collect_vm_values_for_test(test_name, test_params);
if (!Environment::Instance().getCollectResultsOnly()) {
ASSERT_GT(test_refs.references[VMSIZE], 0) << "Reference value of VmSize is less than 0. Value: "
<< test_refs.references[VMSIZE];
ASSERT_GT(test_refs.references[VMPEAK], 0) << "Reference value of VmPeak is less than 0. Value: "
<< test_refs.references[VMPEAK];
ASSERT_GT(test_refs.references[VMRSS], 0) << "Reference value of VmRSS is less than 0. Value: "
<< test_refs.references[VMRSS];
ASSERT_GT(test_refs.references[VMHWM], 0) << "Reference value of VmHWM is less than 0. Value: "
<< test_refs.references[VMHWM];
}
}
};
// tests_pipelines/tests_pipelines.cpp
TEST_P(MemCheckTestSuite, create_exenetwork) {
std::string test_name = "create_exenetwork";
auto test_params = GetParam();
log_info("Create ExecutableNetwork from network: \"" << model
<< "\" for device: \"" << device << "\"");
auto test_pipeline = [&]{
MemCheckPipeline memCheckPipeline;
TestReferences test_refs;
test_refs.collect_vm_values_for_test(test_name, test_params);
Core ie;
ie.GetVersions(device);
CNNNetwork cnnNetwork = ie.ReadNetwork(model);
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
checkRefVmValues();
log_info("Memory consumption after LoadNetwork:");
memCheckPipeline.record_measures(test_name);
TestResult res = test_create_exenetwork(test_params.model_name, test_params.model, test_params.device,
test_refs.ref_vmsize, test_refs.ref_vmpeak, test_refs.ref_vmrss,
test_refs.ref_vmhwm);
log_debug(memCheckPipeline.get_reference_record_for_test(test_name, model_name, device));
return memCheckPipeline.measure();
};
TestResult res = common_test_pipeline(test_pipeline, test_refs.references);
EXPECT_EQ(res.first, TestStatus::TEST_OK) << res.second;
}
TEST_P(MemCheckTestSuite, infer_request_inference) {
std::string test_name = "infer_request_inference";
auto test_params = GetParam();
log_info("Inference of InferRequest from network: \"" << model
<< "\" for device: \"" << device << "\"");
auto test_pipeline = [&]{
MemCheckPipeline memCheckPipeline;
TestReferences test_refs;
test_refs.collect_vm_values_for_test(test_name, test_params);
Core ie;
ie.GetVersions(device);
CNNNetwork cnnNetwork = ie.ReadNetwork(model);
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
InferRequest inferRequest = exeNetwork.CreateInferRequest();
inferRequest.Infer();
OutputsDataMap output_info(cnnNetwork.getOutputsInfo());
for (auto &output : output_info)
Blob::Ptr outputBlob = inferRequest.GetBlob(output.first);
checkRefVmValues();
log_info("Memory consumption after Inference:");
memCheckPipeline.record_measures(test_name);
TestResult res = test_infer_request_inference(test_params.model_name, test_params.model, test_params.device,
test_refs.ref_vmsize, test_refs.ref_vmpeak, test_refs.ref_vmrss,
test_refs.ref_vmhwm);
log_debug(memCheckPipeline.get_reference_record_for_test(test_name, model_name, device));
return memCheckPipeline.measure();
};
TestResult res = common_test_pipeline(test_pipeline, test_refs.references);
EXPECT_EQ(res.first, TestStatus::TEST_OK) << res.second;
}
// tests_pipelines/tests_pipelines.cpp

170
tests/stress_tests/memcheck_tests/tests_pipelines/tests_pipelines.cpp
View File

@ -6,135 +6,79 @@
#include <string>
#include <math.h>
#include <chrono>
#include <inference_engine.hpp>
#define REPORTING_THRESHOLD 1.3
// delimiter used for measurements print. Should be compatible with script parses tests logs
#define MEMCHECK_DELIMITER "\t\t"
using namespace InferenceEngine;
#define getAlignedVmValues(vmsize, vmpeak, vmrss, vmhwm, vmsize_to_align, vmrss_to_align) \
getVmValues(test_cur_vmsize, test_cur_vmpeak, test_cur_vmrss, test_cur_vmhwm); \
test_cur_vmsize -= vmsize_before_test; \
test_cur_vmpeak -= vmsize_before_test; \
test_cur_vmrss -= vmrss_before_test; \
test_cur_vmhwm -= vmrss_before_test;
MemCheckPipeline::MemCheckPipeline() {
start_measures[VMRSS] = (long) getVmRSSInKB();
start_measures[VMHWM] = start_measures[VMRSS];
start_measures[VMSIZE] = (long) getVmSizeInKB();
start_measures[VMPEAK] = start_measures[VMSIZE];
start_measures[THREADS] = (long) getThreadsNum();
}
#define log_debug_ref_record_for_test(test_name) \
log_debug("Record to update reference config: " \
<< "<model path=\"" + model_name + "\"" + " test=\"" + test_name + "\" device=\"" + \
target_device + \
"\" vmsize=\"" + std::to_string((int) (test_cur_vmsize * REPORTING_THRESHOLD)) + \
"\" vmpeak=\"" + std::to_string((int) (test_cur_vmpeak * REPORTING_THRESHOLD)) + \
"\" vmrss=\"" + std::to_string((int) (test_cur_vmrss * REPORTING_THRESHOLD)) + \
"\" vmhwm=\"" + std::to_string((int) (test_cur_vmhwm * REPORTING_THRESHOLD)) + "\" />");
std::array<long, MeasureValueMax> MemCheckPipeline::_measure() {
std::array<long, MeasureValueMax> measures;
measures[VMRSS] = (long) getVmRSSInKB();
measures[VMHWM] = (long) getVmHWMInKB();
measures[VMSIZE] = (long) getVmSizeInKB();
measures[VMPEAK] = (long) getVmPeakInKB();
measures[THREADS] = (long) getThreadsNum(); // TODO: resolve *-32295
return measures;
}
#define log_info_ref_mem_usage() \
log_info("Reference values of virtual memory consumption:"); \
log_info("VMRSS\t\tVMHWM\t\tVMSIZE\t\tVMPEAK"); \
log_info(ref_vmrss << "\t\t" << ref_vmhwm << "\t\t" << ref_vmsize << "\t\t" << ref_vmpeak);
std::array<long, MeasureValueMax> MemCheckPipeline::measure() {
std::array<long, MeasureValueMax> measures = _measure();
std::transform(std::begin(measures), std::end(measures), std::begin(start_measures), std::begin(measures),
[](long measure, long start_measure) -> long {
return measure - start_measure;
});
return measures;
}
#define log_info_cur_mem_usage() \
log_info("Current values of virtual memory consumption:"); \
log_info("VMRSS\t\tVMHWM\t\tVMSIZE\t\tVMPEAK"); \
log_info(test_cur_vmrss << "\t\t" << test_cur_vmhwm << "\t\t" << test_cur_vmsize << "\t\t" << test_cur_vmpeak);
void MemCheckPipeline::record_measures(const std::string & id) {
std::array<long, MeasureValueMax> measures = measure();
log_debug("[ MEASURE ] " << MEMCHECK_DELIMITER << id);
log_info(util::get_measure_values_headers(MEMCHECK_DELIMITER));
log_info(util::get_measure_values_as_str(measures, MEMCHECK_DELIMITER));
}
TestResult
test_create_exenetwork(const std::string &model_name, const std::string &model_path, const std::string &target_device,
const long &ref_vmsize, const long &ref_vmpeak, const long &ref_vmrss, const long &ref_vmhwm) {
log_info("Create ExecutableNetwork from network: \"" << model_path
<< "\" for device: \"" << target_device << "\"");
long vmsize_before_test = 0, vmrss_before_test = 0,
test_cur_vmsize = 0, test_cur_vmpeak = 0,
test_cur_vmrss = 0, test_cur_vmhwm = 0;
std::string MemCheckPipeline::get_reference_record_for_test(std::string test_name, std::string model_name,
std::string target_device) {
std::array<long, MeasureValueMax> measures = measure();
std::stringstream ss;
ss << "Record to update reference config: "
<< "<model path=\"" << model_name << "\"" <<
" test=\"" << test_name << "\" device=\"" << target_device <<
"\" vmsize=\"" << (int) (measures[VMSIZE] * REPORTING_THRESHOLD) <<
"\" vmpeak=\"" << (int) (measures[VMPEAK] * REPORTING_THRESHOLD) <<
"\" vmrss=\"" << (int) (measures[VMRSS] * REPORTING_THRESHOLD) <<
"\" vmhwm=\"" << (int) (measures[VMHWM] * REPORTING_THRESHOLD) << "\" />";
return ss.str();
}
vmsize_before_test = (long) getVmSizeInKB();
vmrss_before_test = (long) getVmRSSInKB();
TestResult common_test_pipeline(const std::function<std::array<long, MeasureValueMax>()>& test_pipeline,
const std::array<long, MeasureValueMax> &references) {
log_info("Reference values of virtual memory consumption:");
log_info(util::get_measure_values_headers(MEMCHECK_DELIMITER));
log_info(util::get_measure_values_as_str(references, MEMCHECK_DELIMITER));
Core ie;
CNNNetwork cnnNetwork = ie.ReadNetwork(model_path);
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, target_device);
std::array<long, MeasureValueMax> measures = test_pipeline();
getAlignedVmValues(test_cur_vmsize, test_cur_vmpeak, test_cur_vmrss, test_cur_vmhwm,
vmsize_before_test, vmrss_before_test);
log_debug_ref_record_for_test("create_exenetwork");
log_info_ref_mem_usage();
log_info_cur_mem_usage();
if ((!Environment::Instance().getCollectResultsOnly()) && (test_cur_vmrss > ref_vmrss))
if ((!Environment::Instance().getCollectResultsOnly()) && (measures[VMRSS] > references[VMRSS]))
return TestResult(TestStatus::TEST_FAILED,
"Test failed: RSS virtual memory consumption became greater than reference.\n"
"Reference RSS memory consumption: " + std::to_string(ref_vmrss) + " KB.\n" +
"Current RSS memory consumption: " + std::to_string(test_cur_vmrss) + " KB.\n");
"Reference RSS memory consumption: " + std::to_string(references[VMRSS]) + " KB.\n" +
"Current RSS memory consumption: " + std::to_string(measures[VMRSS]) + " KB.\n");
if ((!Environment::Instance().getCollectResultsOnly()) && (test_cur_vmhwm > ref_vmhwm))
if ((!Environment::Instance().getCollectResultsOnly()) && (measures[VMHWM] > references[VMHWM]))
return TestResult(TestStatus::TEST_FAILED,
"Test failed: HWM (peak of RSS) virtual memory consumption is greater than reference.\n"
"Reference HWM of memory consumption: " + std::to_string(ref_vmhwm) + " KB.\n" +
"Current HWM of memory consumption: " + std::to_string(test_cur_vmhwm) + " KB.\n");
return TestResult(TestStatus::TEST_OK, "");
}
TestResult
test_infer_request_inference(const std::string &model_name, const std::string &model_path,
const std::string &target_device,
const long &ref_vmsize, const long &ref_vmpeak, const long &ref_vmrss,
const long &ref_vmhwm) {
log_info("Inference of InferRequest from network: \"" << model_path
<< "\" for device: \"" << target_device << "\"");
long vmsize_before_test = 0, vmrss_before_test = 0,
test_cur_vmsize = 0, test_cur_vmpeak = 0,
test_cur_vmrss = 0, test_cur_vmhwm = 0;
std::chrono::system_clock::time_point t_start, t_end;
std::chrono::duration<double> t_diff;
vmsize_before_test = (long) getVmSizeInKB();
vmrss_before_test = (long) getVmRSSInKB();
Core ie;
CNNNetwork cnnNetwork = ie.ReadNetwork(model_path);
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, target_device);
InferRequest infer_request = exeNetwork.CreateInferRequest();
log_info_ref_mem_usage();
t_start = std::chrono::system_clock::now();
int seconds = 1;
do {
infer_request.Infer();
OutputsDataMap output_info(cnnNetwork.getOutputsInfo());
for (auto &output : output_info)
Blob::Ptr outputBlob = infer_request.GetBlob(output.first);
t_end = std::chrono::system_clock::now();
t_diff = t_end - t_start;
getAlignedVmValues(test_cur_vmsize, test_cur_vmpeak, test_cur_vmrss, test_cur_vmhwm,
vmsize_before_test, vmrss_before_test);
if (t_diff.count() > (double) (seconds)) {
log_info("Current values of virtual memory consumption after " << seconds << " seconds:");
log_info("VMRSS\t\tVMHWM\t\tVMSIZE\t\tVMPEAK");
log_info(test_cur_vmrss << "\t\t" << test_cur_vmhwm << "\t\t" << test_cur_vmsize << "\t\t" << test_cur_vmpeak);
seconds++;
}
} while (t_diff.count() < 5);
log_debug_ref_record_for_test("infer_request_inference");
if ((!Environment::Instance().getCollectResultsOnly()) && (test_cur_vmrss > ref_vmrss))
return TestResult(TestStatus::TEST_FAILED,
"Test failed: RSS virtual memory consumption became greater than reference.\n"
"Reference RSS memory consumption: " + std::to_string(ref_vmrss) + " KB.\n" +
"Current RSS memory consumption: " + std::to_string(test_cur_vmrss) + " KB.\n");
if ((!Environment::Instance().getCollectResultsOnly()) && (test_cur_vmhwm > ref_vmhwm))
return TestResult(TestStatus::TEST_FAILED,
"Test failed: HWM (peak of RSS) virtual memory consumption is greater than reference.\n"
"Reference HWM of memory consumption: " + std::to_string(ref_vmhwm) + " KB.\n" +
"Current HWM of memory consumption: " + std::to_string(test_cur_vmhwm) + " KB.\n");
"Reference HWM of memory consumption: " + std::to_string(references[VMHWM]) + " KB.\n" +
"Current HWM of memory consumption: " + std::to_string(measures[VMHWM]) + " KB.\n");
return TestResult(TestStatus::TEST_OK, "");
}

50
tests/stress_tests/memcheck_tests/tests_pipelines/tests_pipelines.h
View File

@ -4,14 +4,56 @@
#pragma once
#include "../tests_utils.h"
#include "../../common/tests_utils.h"
#include "../../common/utils.h"
#include <string>
// tests_pipelines/tests_pipelines.cpp
TestResult test_create_exenetwork(const std::string &model_name, const std::string &model_path, const std::string &target_device,
const long &ref_vmsize, const long &ref_vmpeak, const long &ref_vmrss, const long &ref_vmhwm);
TestResult test_infer_request_inference(const std::string &model_name, const std::string &model_path, const std::string &target_device,
const long &ref_vmsize, const long &ref_vmpeak, const long &ref_vmrss, const long &ref_vmhwm);
/**
* @brief Class response for encapsulating measure and measurements printing
*
* Current class measures only in scope of it's lifetime. In this case need
* to note that deletion of objects created before class creation may lead
* to negative values because of alignment on starting values.
* Also deletion of objects created in scope of class lifetime may decrease
* values computed on previous measure.
*/
class MemCheckPipeline {
private:
std::array<long, MeasureValueMax> start_measures; // measures before run (will be used as baseline)
/**
* @brief Measures values at the current point of time
*/
std::array<long, MeasureValueMax> _measure();
public:
/**
* @brief Constructs MemCheckPipeline object and
* measure values to use as baseline
*/
MemCheckPipeline();
/**
* @brief Measures values at the current point of time and
* returns measurements aligned on a baseline
*/
std::array<long, MeasureValueMax> measure();
/**
* @brief Measures values and records aligned measurements using provided identifier
* provided identifier
*/
void record_measures(const std::string & id);
/**
* @brief Prepares string used for fast generation of file with references
*/
std::string get_reference_record_for_test(std::string test_name, std::string model_name,
std::string target_device);
};
TestResult common_test_pipeline(const std::function<std::array<long, MeasureValueMax>()>& test_pipeline,
const std::array<long, MeasureValueMax> &references);
// tests_pipelines/tests_pipelines.cpp

37
tests/stress_tests/memcheck_tests/tests_utils.h
View File

@ -2,10 +2,35 @@
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "../common/tests_utils.h"
#include <pugixml.hpp>
// Measure values
enum MeasureValue { VMRSS = 0, VMHWM, VMSIZE, VMPEAK, THREADS, MeasureValueMax };
// Measure values headers
const std::array<std::string, MeasureValueMax> MeasureValueHeader { "VMRSS", "VMHWM", "VMSIZE", "VMPEAK", "THREADS" };
namespace util {
template <typename Type>
static std::string get_measure_values_as_str(const std::array<Type, MeasureValueMax> & array,
const std::string & delimiter = "\t\t") {
std::string str = std::to_string(*array.begin());
for (auto it = array.begin() + 1; it != array.end(); it++)
str += delimiter + std::to_string(*it);
return str;
}
static std::string get_measure_values_headers(const std::string & delimiter = "\t\t") {
std::string str = *MeasureValueHeader.begin();
for (auto it = MeasureValueHeader.begin() + 1; it != MeasureValueHeader.end(); it++)
str += delimiter + *it;
return str;
}
}
class MemCheckEnvironment {
private:
pugi::xml_document _refs_config;
@ -32,9 +57,11 @@ private:
std::vector<std::string> model_path_v, test_name_v, device_v;
std::vector<long> vmsize_v, vmpeak_v, vmrss_v, vmhwm_v;
public:
long ref_vmsize = -1, ref_vmpeak = -1, ref_vmrss = -1, ref_vmhwm = -1;
std::array<long, MeasureValueMax> references;
TestReferences () {
std::fill(references.begin(), references.end(), -1);
// Parse RefsConfig from MemCheckEnvironment
std::string models_path = Environment::Instance().getEnvConfig()
.child("attributes").child("irs_path").child("value").text().as_string();
@ -68,10 +95,10 @@ public:
if (test_name_v[i] == test_name)
if (model_path_v[i] == test_params.model)
if (device_v[i] == test_params.device) {
ref_vmsize = vmsize_v[i];
ref_vmpeak = vmpeak_v[i];
ref_vmrss = vmrss_v[i];
ref_vmhwm = vmhwm_v[i];
references[VMSIZE] = vmsize_v[i];
references[VMPEAK] = vmpeak_v[i];
references[VMRSS] = vmrss_v[i];
references[VMHWM] = vmhwm_v[i];
}
}
};

6
tests/stress_tests/scripts/memcheck_upload.py
View File

@ -30,8 +30,7 @@ DB_COLLECTIONS = ["commit", "nightly", "weekly"]
PRODUCT_NAME = 'dldt' # product name from build manifest
RE_GTEST_MODEL_XML = re.compile(r'<model[^>]*>')
RE_GTEST_CUR_MEASURE = re.compile(
r'Current values of virtual memory consumption')
RE_GTEST_CUR_MEASURE = re.compile(r'\[\s*MEASURE\s*\]')
RE_GTEST_REF_MEASURE = re.compile(
r'Reference values of virtual memory consumption')
RE_GTEST_PASSED = re.compile(r'\[\s*PASSED\s*\]')
@ -99,13 +98,14 @@ def parse_memcheck_log(log_path):
ref_metrics = dict(zip(heading, values))
for index in reversed(range(len(log_lines))):
if RE_GTEST_CUR_MEASURE.search(log_lines[index]):
test_name = log_lines[index].split()[-1]
heading = [name.lower() for name in log_lines[index+1]
[len(GTEST_INFO):].split()]
values = [int(val) for val in log_lines[index+2]
[len(GTEST_INFO):].split()]
entry = SimpleNamespace(
metrics=dict(zip(heading, values)),
test_name=model['test'],
test_name=test_name,
model_name=os.path.splitext(
os.path.basename(model['path']))[0],
precision=next(pr for pr in PRECISSIONS if pr.upper()