IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CONFORMANCE][IE TESTS] Reduce SLT execution time (#14165)

Browse Source 
* init

* Reduce time

* remove extra

* try

* dd

* one more try

* Fix mistakes

* skip some cpu tests

* comment

* one more skip
This commit is contained in:
Irina Efode
2022-11-29 14:49:03 +03:00
committed by GitHub
parent 5a59b2b8f2
commit 48dd92de47
7 changed files with 127 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp
View File

@@ -175,7 +175,7 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*OVClassLoadNetworkTest.*QueryNetwork(MULTIWithHETERO|HETEROWithMULTI)NoThrow_V10.*)",
R"(.*OVClassNetworkTestP.*QueryNetworkMultiThrows.*)",
R"(.*CachingSupportCase.*LoadNetworkCacheTestBase.*(TIwithLSTMcell1|MatMulBias|2InputSubtract)_(i|u).*)",
R"(.*CachingSupportCase.*LoadNetworkCacheTestBase.*ReadConcatSplitAssign.*)",
R"(.*CachingSupportCase.*ReadConcatSplitAssign.*)",
R"(.*IEClassQueryNetworkTest.*QueryNetwork.*)",
R"(.*IEClassLoadNetworkTest.*(Load|Query)Network.*)",
// Issue: 95239
@@ -191,6 +191,7 @@ std::vector<std::string> disabledTestPatterns() {
R"(smoke_dynamic/AUGRUCellCPUTest.CompareWithRefs/IS=\(\[\?\.1\]_\[\?\.1\]_\[\?\.1\]_\)_TS=\{\(1\.1\)_\(1\.1\)_\(1\.1\)\}_\{\(3\.1\)_\(3\.1\)_\(3\.1\)\}_\{\(5\.1\)_\(5\.1\)_\(5\.1\)\}_decompose=0_activations=\(sigmoid\.tanh\)_clip=0_linear=0_netPrec=f32__inFmts=nc\.nc_outFmts=nc_primitive=ref_any_PluginConf_ENFORCE_BF16=YES)", // NOLINT
R"(smoke_dynamic/GRUCellCPUTest.CompareWithRefs/IS=\(\[\?.1\]_\[\?\.1\]_\)_TS=\{\(1\.1\)_\(1\.1\)\}_\{\(3\.1\)_\(3\.1\)\}_\{\(5\.1\)_\(5\.1\)\}_decompose=0_activations=\(sigmoid\.tanh\)_clip=0_linear=0_netPrec=f32__inFmts=nc\.nc_outFmts=nc_primitive=ref_any_PluginConf_ENFORCE_BF16=YES)", // NOLINT
R"(nightly_dynamic_bf16/RNNSequenceCPUTest.*activations=\(relu\).*)",
R"(smoke_dynamic_BatchSizeOne/RNNSequenceCPUTest.*IS=\(\[1\.\?\.10\]_\[1\.1\.10\]_\[\?\]_\)_TS=\{\(1\.2\.10\)_\(1\.1\.10\)_\(1\)\}_\{\(1\.4\.10\)_\(1\.1\.10\)_\(1\)\}_\{\(1\.8\.10\)_\(1\.1\.10\)_\(1\)\}_seqMode=PURE_SEQ_activations=\(relu\)_clip=0_direction=forward_netPrec=f32__inFmts=ncw\.ntc_outFmts=ncw\.ncw_primitive=ref_any)", // NOLINT
};
#define FIX_62820 0

10
src/plugins/intel_cpu/tests/functional/single_layer_tests/adaptive_pooling.cpp
View File

@@ -133,6 +133,7 @@ protected:
}
void validate() override {
auto actualOutputs = get_plugin_outputs();
if (function->get_parameters().size() == 2) {
auto pos = std::find_if(inputs.begin(), inputs.end(),
[](const std::pair<std::shared_ptr<ov::Node>, ov::Tensor> &params) {
@@ -141,7 +142,14 @@ protected:
IE_ASSERT(pos != inputs.end());
inputs.erase(pos);
}
SubgraphBaseTest::validate();
auto expectedOutputs = calculate_refs();
if (expectedOutputs.empty()) {
return;
}
ASSERT_EQ(actualOutputs.size(), expectedOutputs.size())
<< "nGraph interpreter has " << expectedOutputs.size() << " outputs, while IE " << actualOutputs.size();
compare(expectedOutputs, actualOutputs);
}
void generate_inputs(const std::vector<ov::Shape>& targetInputStaticShapes) override {

10
src/plugins/intel_cpu/tests/functional/single_layer_tests/convolution_backprop_data.cpp
View File

@@ -122,6 +122,7 @@ public:
}
void validate() override {
auto actualOutputs = get_plugin_outputs();
if (function->get_parameters().size() == 2) {
auto pos = std::find_if(inputs.begin(), inputs.end(),
[](const std::pair<std::shared_ptr<ov::Node>, ov::Tensor> &params) {
@@ -130,7 +131,14 @@ public:
IE_ASSERT(pos != inputs.end());
inputs.erase(pos);
}
SubgraphBaseTest::validate();
auto expectedOutputs = calculate_refs();
if (expectedOutputs.empty()) {
return;
}
ASSERT_EQ(actualOutputs.size(), expectedOutputs.size())
<< "nGraph interpreter has " << expectedOutputs.size() << " outputs, while IE " << actualOutputs.size();
compare(expectedOutputs, actualOutputs);
}
void configure_model() override {

10
src/plugins/intel_cpu/tests/functional/single_layer_tests/group_convolution_backprop_data.cpp
View File

@@ -123,6 +123,7 @@ public:
}
void validate() override {
auto actualOutputs = get_plugin_outputs();
if (function->get_parameters().size() == 2) {
auto pos = std::find_if(inputs.begin(), inputs.end(),
[](const std::pair<std::shared_ptr<ov::Node>, ov::Tensor> &params) {
@@ -131,7 +132,14 @@ public:
IE_ASSERT(pos != inputs.end());
inputs.erase(pos);
}
SubgraphBaseTest::validate();
auto expectedOutputs = calculate_refs();
if (expectedOutputs.empty()) {
return;
}
ASSERT_EQ(actualOutputs.size(), expectedOutputs.size())
<< "nGraph interpreter has " << expectedOutputs.size() << " outputs, while IE " << actualOutputs.size();
compare(expectedOutputs, actualOutputs);
}
void configure_model() override {

10
src/plugins/intel_cpu/tests/functional/single_layer_tests/one_hot.cpp
View File

@@ -110,6 +110,7 @@ protected:
ngraph::helpers::resize_function(funcRef, targetInputStaticShapes);
}
void validate() override {
auto actualOutputs = get_plugin_outputs();
if (function->get_parameters().size() == 2) {
auto pos = std::find_if(inputs.begin(), inputs.end(),
[](const std::pair<std::shared_ptr<ov::Node>, ov::Tensor> &params) {
@@ -118,7 +119,14 @@ protected:
IE_ASSERT(pos != inputs.end());
inputs.erase(pos);
}
SubgraphBaseTest::validate();
auto expectedOutputs = calculate_refs();
if (expectedOutputs.empty()) {
return;
}
ASSERT_EQ(actualOutputs.size(), expectedOutputs.size())
<< "nGraph interpreter has " << expectedOutputs.size() << " outputs, while IE " << actualOutputs.size();
compare(expectedOutputs, actualOutputs);
}
std::shared_ptr<ngraph::Function> createFunction(bool depthConst) {
auto params = ngraph::builder::makeDynamicParams(ngraph::element::i32, {inputDynamicShapes.front()});

23
src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp
View File

@@ -3,8 +3,10 @@
//
#include <signal.h>
#include <setjmp.h>
#include <fstream>
#include "transformations/convert_precision.hpp"
#include <thread>
#ifdef _WIN32
#include <process.h>
@@ -12,6 +14,7 @@
#include "openvino/core/preprocess/pre_post_process.hpp"
#include "openvino/pass/serialize.hpp"
#include "transformations/convert_precision.hpp"
#include "common_test_utils/graph_comparator.hpp"
@@ -19,14 +22,13 @@
#include "common_test_utils/file_utils.hpp"
#include "common_test_utils/crash_handler.hpp"
#include <common_test_utils/ov_tensor_utils.hpp>
#include "common_test_utils/ov_tensor_utils.hpp"
#include "functional_test_utils/skip_tests_config.hpp"
#include "shared_test_classes/base/ov_subgraph.hpp"
#include "shared_test_classes/base/utils/generate_inputs.hpp"
#include "shared_test_classes/base/utils/compare_results.hpp"
#include <setjmp.h>
namespace ov {
namespace test {
@@ -78,7 +80,6 @@ void SubgraphBaseTest::run() {
throw std::runtime_error("Incorrect target static shape: " +
CommonTestUtils::vec2str(targetStaticShapeVec) + " " + ex.what());
}
infer();
validate();
}
status = ov::test::utils::PassRate::Statuses::PASSED;
@@ -314,6 +315,7 @@ std::vector<ov::Tensor> SubgraphBaseTest::calculate_refs() {
}
std::vector<ov::Tensor> SubgraphBaseTest::get_plugin_outputs() {
infer();
auto outputs = std::vector<ov::Tensor>{};
for (const auto& output : function->outputs()) {
outputs.push_back(inferRequest.get_tensor(output));
@@ -322,8 +324,17 @@ std::vector<ov::Tensor> SubgraphBaseTest::get_plugin_outputs() {
}
void SubgraphBaseTest::validate() {
auto expectedOutputs = calculate_refs();
const auto& actualOutputs = get_plugin_outputs();
std::vector<ov::Tensor> expectedOutputs, actualOutputs;
#ifndef NDEBUG
actualOutputs = get_plugin_outputs();
expectedOutputs = calculate_refs();
#else
std::thread t_device([&]{ actualOutputs = get_plugin_outputs(); });
std::thread t_ref([&]{ expectedOutputs = calculate_refs(); });
t_device.join();
t_ref.join();
#endif
if (expectedOutputs.empty()) {
return;

125
src/tests/ie_test_utils/common_test_utils/ov_tensor_utils.cpp
View File

@@ -173,6 +173,49 @@ tensor.get_size(), range, start_from, resolution); break;
return tensor;
}
constexpr double eps = std::numeric_limits<double>::epsilon();
inline double less(double a, double b) {
return (b - a) > (std::fmax(std::fabs(a), std::fabs(b)) * eps);
}
inline double less_or_equal(double a, double b) {
return (b - a) >= (std::fmax(std::fabs(a), std::fabs(b)) * eps);
}
struct Error {
double max = 0.;
double mean = 0.;
size_t max_coordinate = 0;
size_t count = 0;
double threshold;
Error(double _threshold) : threshold(_threshold) {}
void update(double val, size_t i) {
if (less(max, val)) {
max = val;
max_coordinate = i;
}
mean += val;
count += less(threshold, val);
}
};
inline double calculate_median(std::vector<double>& abs_values) {
double abs_median = 0.;
auto expected_shape = abs_values.size();
if (expected_shape % 2) {
std::nth_element(abs_values.begin(), abs_values.begin() + expected_shape / 2, abs_values.end());
abs_median = abs_values[expected_shape / 2];
} else {
std::nth_element(abs_values.begin(), abs_values.begin() + expected_shape / 2, abs_values.end());
std::nth_element(abs_values.begin(), abs_values.begin() + (expected_shape - 1) / 2, abs_values.end());
abs_median = (abs_values[(expected_shape - 1) / 2] + abs_values[expected_shape / 2]) / 2.0;
}
return abs_median;
}
template<typename ExpectedT, typename ActualT>
void compare(const ov::Tensor& expected,
const ov::Tensor& actual,
@@ -185,7 +228,6 @@ void compare(const ov::Tensor& expected,
out_stream << "Expected and actual shape are different: " << expected_shape << " " << actual_shape;
throw std::runtime_error(out_stream.str());
}
if (shape_size(actual_shape) == 0) {
return;
}
@@ -194,57 +236,33 @@ void compare(const ov::Tensor& expected,
auto actual_data = actual.data<ActualT>();
double abs_threshold = abs_threshold_;
double rel_threshold = rel_threshold_;
size_t shape_size_cnt = shape_size(expected_shape);
if (abs_threshold == std::numeric_limits<double>::max() && rel_threshold == std::numeric_limits<double>::max()) {
if (sizeof(ExpectedT) == 1 || sizeof(ActualT) == 1) {
abs_threshold = 1.;
} else {
std::vector<double> abs_values;
abs_values.reserve(shape_size(expected_shape));
for (size_t i = 0; i < shape_size(expected_shape); i++) {
abs_values.push_back(std::fabs(static_cast<double>(expected_data[i])));
std::vector<double> abs_values(shape_size_cnt);
for (size_t i = 0; i < shape_size_cnt; i++) {
abs_values[i] = std::fabs(static_cast<double>(expected_data[i]));
}
std::sort(abs_values.begin(), abs_values.end());
double abs_median;
if (abs_values.size() % 2 == 0) {
abs_median = abs_values.size() > 2 ?
(abs_values.at(abs_values.size()/2) + abs_values.at(abs_values.size()/2 + 1))/2 : (abs_values.front() + abs_values.back())/2;
} else {
abs_median = abs_values.at(abs_values.size()/2);
}
abs_threshold = abs_median == 0.f ? 1e-5 : 0.05 * abs_median;
auto abs_median = calculate_median(abs_values);
abs_threshold = abs_median * 0.05 < 1e-5 ? 1e-5 : 0.05 * abs_median;
if (std::is_integral<ExpectedT>::value) {
abs_threshold = std::ceil(abs_threshold);
}
}
}
if (!std::isnan(abs_threshold) && !std::isnan(rel_threshold)) {
std::cout << "abs_threshold: " << abs_threshold << " rel_threshold: " << rel_threshold << std::endl;
if (!std::isnan(abs_threshold)) {
std::cout << " rel_threshold: " << rel_threshold << std::endl;
}
struct Error {
double max = 0.;
double mean = 0.;
size_t max_coordinate = 0;
size_t count = 0;
} abs_error, rel_error;
auto less = [] (double a, double b) {
auto eps = std::numeric_limits<double>::epsilon();
return (b - a) > (std::fmax(std::fabs(a), std::fabs(b)) * eps);
};
auto less_or_equal = [] (double a, double b) {
auto eps = std::numeric_limits<double>::epsilon();
return (b - a) >= (std::fmax(std::fabs(a), std::fabs(b)) * eps);
};
for (size_t i = 0; i < shape_size(expected_shape); i++) {
if (!std::isnan(rel_threshold)) {
std::cout << "abs_threshold: " << abs_threshold << std::endl;
}
Error abs_error(abs_threshold), rel_error(rel_threshold);
for (size_t i = 0; i < shape_size_cnt; ++i) {
double expected_value = expected_data[i];
double actual_value = actual_data[i];
auto error = [&] (Error& err, double val, double threshold) {
if (less(err.max, val)) {
err.max = val;
err.max_coordinate = i;
}
err.mean += val;
err.count += less(threshold, val);
};
if (std::isnan(expected_value)) {
std::ostringstream out_stream;
out_stream << "Expected value is NAN on coordinate: " << i;
@@ -255,24 +273,25 @@ void compare(const ov::Tensor& expected,
out_stream << "Actual value is NAN on coordinate: " << i;
throw std::runtime_error(out_stream.str());
}
auto abs = std::fabs(expected_value - actual_value);
auto rel = expected_value ? (abs/std::fabs(expected_value)) : abs;
error(abs_error, abs, abs_threshold);
error(rel_error, rel, rel_threshold);
double abs = std::fabs(expected_value - actual_value);
double rel = expected_value ? (abs / std::fabs(expected_value)) : abs;
abs_error.update(abs, i);
rel_error.update(rel, i);
}
abs_error.mean /= shape_size(expected_shape);
rel_error.mean /= shape_size(expected_shape);
abs_error.mean /= shape_size_cnt;
rel_error.mean /= shape_size_cnt;
if (!(less_or_equal(abs_error.max, abs_threshold) && less_or_equal(rel_error.max, rel_threshold))) {
std::ostringstream out_stream;
out_stream << "abs_max < abs_threshold && rel_max < rel_threshold" <<
"\n\t abs_max: " << abs_error.max <<
"\n\t\t coordinate " << abs_error.max_coordinate<<
"; abs errors count " << abs_error.count << "; abs mean " <<
abs_error.mean << "; abs threshold " << abs_threshold <<
"\n\t rel_max: " << rel_error.max <<
"\n\t\t coordinate " << rel_error.max_coordinate <<
"; rel errors count " << rel_error.count << "; rel mean " <<
rel_error.mean << "; rel threshold " << rel_threshold;
"\n\t abs_max: " << abs_error.max <<
"\n\t\t coordinate " << abs_error.max_coordinate<<
"; abs errors count " << abs_error.count << "; abs mean " <<
abs_error.mean << "; abs threshold " << abs_threshold <<
"\n\t rel_max: " << rel_error.max <<
"\n\t\t coordinate " << rel_error.max_coordinate <<
"; rel errors count " << rel_error.count << "; rel mean " <<
rel_error.mean << "; rel threshold " << rel_threshold;
throw std::runtime_error(out_stream.str());
}
}