IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[IE TESTS] Allow to specify skip tests config by list file (#5388)

Browse Source 
* [IE TESTS] Allow to specify skip tests config by list file

* Fix func name

* Fix includes

* TEmplate
This commit is contained in:
Irina Efode 2021-04-28 11:02:26 +03:00 committed by GitHub
parent 08ca1d4307
commit 60ac91b446
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 270 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
inference-engine/tests/functional/plugin/conformance/test_runner/README.md
View File

@ -22,6 +22,7 @@ The target is able to take the following command-line arguments:
* `--input_folders` specifies folders with IRs to run. The separator is `,`.
* `--plugin_lib_name` is name of plugin library. The example is MKLDNNPlugin. Use only with unregistered in IE Core devices.
* `--disable_test_config` allows to ignore all skipped tests with the exception of `DISABLED_` prefix using.
* `--skip_config_path` allows to specify paths to files contain regular expressions list to skip tests.
* `--extend_report` allows not to re-write device results to the report (add results of this run to the existing). Mutually exclusive with --report_unique_name.
* `--report_unique_name` allows to save report with unique name (report_pid_timestamp.xml). Mutually exclusive with --extend_report.
* `--save_report_timeout` allows to try to save report in cycle using timeout (in seconds).

1
inference-engine/tests/functional/plugin/conformance/test_runner/include/conformance.hpp
View File

@ -7,5 +7,6 @@ namespace ConformanceTests {
extern const char* targetDevice;
extern const char* targetPluginName;
extern std::vector<std::string> IRFolderPaths;
extern std::vector<std::string> disabledTests;
} // namespace ConformanceTests

4
inference-engine/tests/functional/plugin/conformance/test_runner/include/gflag_config.hpp
View File

@ -23,12 +23,15 @@ static const char report_unique_name_message[] = "Optional. Allow to save report
"Mutually exclusive with --extend_report. Default value is false";
static const char save_report_timeout_message[] = "Optional. Allow to try to save report in cycle using timeout (in seconds). "
"Default value is 60 seconds";
static const char skip_config_path_message[] = "Optional. Allows to specify paths to files contain regular expressions list to skip tests. "
"Delimiter is `,` symbol. Default value is empty string.";
DEFINE_bool(h, false, help_message);
DEFINE_string(device, "CPU", target_device_message);
DEFINE_string(plugin_lib_name, "", target_plugin_message);
DEFINE_string(input_folders, ".", input_folders_message);
DEFINE_string(output_folder, ".", output_folder_message);
DEFINE_string(skip_config_path, "", skip_config_path_message);
DEFINE_uint32(save_report_timeout, 60, save_report_timeout_message);
DEFINE_bool(disable_test_config, true, disable_test_config_message);
DEFINE_bool(extend_report, false, extend_report_config_message);
@ -44,6 +47,7 @@ static void showUsage() {
std::cout << std::endl;
std::cout << " -h " << help_message << std::endl;
std::cout << " --disable_test_config " << disable_test_config_message << std::endl;
std::cout << " --skip_config_path \"<paths>\" " << disable_test_config_message << std::endl;
std::cout << " --extend_report " << extend_report_config_message << std::endl;
std::cout << " --report_unique_name " << extend_report_config_message << std::endl;
std::cout << " --save_report_timeout " << extend_report_config_message << std::endl;

4
inference-engine/tests/functional/plugin/conformance/test_runner/skip_configs/skip_config_cpu.lst Normal file
View File

@ -0,0 +1,4 @@
# Hung:
.*AvgPool_1199829.*
.*AvgPool_1201153.*
.*ROIPooling_1199827.*

13
inference-engine/tests/functional/plugin/conformance/test_runner/skip_configs/skip_config_gna.lst Normal file
View File

@ -0,0 +1,13 @@
# Lost results
.*Add_1087636.*
.*Add_2868.*
.*Add_2979.*
.*Add_53543.*
# hung
.*Concat_535028.*
.*Concat_377139.*
.*Concat_379481.*
.*Concat_539044.*
.*Concat_539074.*
.*Concat_534956.*

59
inference-engine/tests/functional/plugin/conformance/test_runner/skip_configs/skip_config_gpu.lst Normal file
View File

@ -0,0 +1,59 @@
# TODO: Issue: 53062
.*TensorIterator_1195103.*
.*TensorIterator_1195129.*
.*TensorIterator_1653010.*
.*TensorIterator_199835.*
.*TensorIterator_303641.*
.*TensorIterator_303667.*
.*TensorIterator_337260.*
.*TensorIterator_362846.*
.*TensorIterator_362869.*
.*TensorIterator_362906.*
.*TensorIterator_365496.*
.*TensorIterator_365522.*
.*TensorIterator_365556.*
.*TensorIterator_365579.*
.*TensorIterator_616544.*
.*TensorIterator_616570.*
.*TensorIterator_864978.*
.*TensorIterator_865004.*
.*TensorIterator_865030.*
.*TensorIterator_865056.*
.*TensorIterator_865082.*
.*TensorIterator_865108.*
.*TensorIterator_972832.*
.*TensorIterator_972858.*
.*TensorIterator_972884.*
.*TensorIterator_972910.*
.*TensorIterator_972936.*
.*TensorIterator_972962.*
.*TensorIterator_972988.*
.*TensorIterator_973014.*
.*TensorIterator_1194007.*
.*TensorIterator_1194033.*
.*TensorIterator_302500.*
.*TensorIterator_337024.*
.*TensorIterator_361044.*
.*TensorIterator_361067.*
.*TensorIterator_361108.*
.*TensorIterator_361131.*
.*TensorIterator_364171.*
.*TensorIterator_364197.*
.*TensorIterator_364231.*
.*TensorIterator_364254.*
.*TensorIterator_615626.*
.*TensorIterator_615652.*
.*TensorIterator_865760.*
.*TensorIterator_865812.*
.*TensorIterator_865838.*
.*TensorIterator_865864.*
.*TensorIterator_973781.*
.*TensorIterator_973807.*
# Hung:
.*AvgPool_1199829.*
.*AvgPool_1201153.*
.*GroupConvolution_330567.*
.*ROIPooling_1199827.*
.*MaxPool_43108.*

146
inference-engine/tests/functional/plugin/conformance/test_runner/skip_configs/skip_config_myriad.lst Normal file
View File

@ -0,0 +1,146 @@
# TODO: Issue: 53061
.*ReduceMean_334003.*
.*ReduceMean_334702.*
.*Select_1172002.*
.*Select_1178458.*
.*Divide_567867.*
.*Divide_1377278.*
.*Divide_1030260.*
.*Pad_274459.*
.*Pad_276201.*
.*Pad_276193.*
.*Add_1070296.*
.*Add_284191.*
.*Add_1087636.*
.*VariadicSplit_614428.*
.*VariadicSplit_642662.*
.*VariadicSplit_412430.*
.*VariadicSplit_475191.*
.*VariadicSplit_475203.*
.*VariadicSplit_279359.*
.*VariadicSplit_31490.*
.*VariadicSplit_1444989.*
.*VariadicSplit_31499.*
.*VariadicSplit_279373.*
.*VariadicSplit_279345.*
.*VariadicSplit_31508.*
.*VariadicSplit_535004.*
.*VariadicSplit_148861.*
.*VariadicSplit_475199.*
.*VariadicSplit_412439.*
.*VariadicSplit_412448.*
.*VariadicSplit_475175.*
.*VariadicSplit_427269.*
.*VariadicSplit_1070300.*
.*VariadicSplit_478455.*
.*VariadicSplit_416024.*
.*VariadicSplit_478469.*
.*VariadicSplit_416006.*
.*VariadicSplit_35108.*
.*VariadicSplit_478473.*
.*VariadicSplit_416015.*
.*VariadicSplit_280446.*
.*VariadicSplit_6686.*
.*VariadicSplit_1087640.*
.*VariadicSplit_1443292.*
.*VariadicSplit_35099.*
.*VariadicSplit_280460.*
.*VariadicSplit_35090.*
.*VariadicSplit_430827.*
.*VariadicSplit_478461.*
.*VariadicSplit_280432.*
.*VariadicSplit_539066.*
.*VariadicSplit_149040.*
.*VariadicSplit_614761.*
.*VariadicSplit_629649.*
.*Unsqueeze_65214.*
.*Unsqueeze_4838.*
.*Unsqueeze_6661.*
.*Multiply_65211.*
.*Multiply_4833.*
.*Multiply_48331.*
.*Multiply_6656.*
.*Subtract_534968.*
.*Subtract_567907.*
.*Gather_21153.*
.*Gather_302464.*
.*Gather_588814.*
.*Gather_588810.*
.*Gather_952843.*
.*Gather_588818.*
.*Gather_588871.*
.*Gather_48276.*
.*Gather_1652589.*
.*Gather_535000.*
.*Gather_360953.*
.*Gather_588867.*
.*Gather_48283.*
.*Gather_1311696.*
.*Gather_588796.*
.*Gather_4847.*
.*Gather_4803.*
.*Gather_284209.*
.*Gather_4869.*
.*Gather_97426.*
.*Gather_360963.*
.*Gather_1570871.*
.*Gather_513092.*
.*Gather_629616.*
.*Gather_48287.*
.*Gather_284215.*
.*Gather_48335.*
.*Gather_1444.*
.*Gather_4810.*
.*Gather_1395.*
.*Gather_1653002.*
.*Gather_53520.*
.*Gather_567864.*
.*Gather_362776.*
.*Gather_567856.*
.*Gather_567919.*
.*Gather_567830.*
.*Gather_567923.*
.*Gather_567860.*
.*Gather_539062.*
.*Gather_1304430.*
.*Gather_938915.*
.*Gather_53527.*
.*Convert_284211.*
.*Convert_4830.*
.*Convert_567844.*
.*Convert_6653.*
.*FakeQuantize_738613.*
.*FakeQuantize_738639.*
.*Split_65164.*
.*Split_274446.*
.*Split_361038.*
.*Split_330570.*
.*Split_276188.*
.*Split_68625.*
.*Split_333279.*
.*Split_362843.*
.*Split_68647.*
.*Floor_6658.*
.*TopK_1172093.*
.*TopK_1172038.*
.*TopK_1178526.*
.*TopK_515250.*
.*TopK_1178477.*
.*Reshape_1105704.*
.*Reshape_1118646.*
.*Reshape_588858.*
.*Reshape_567910.*
# TODO: Crashes: Should be handled
.*CTCGreedyDecoderSeqLen_271798.*
.*MVN_276196.*
.*MVN_274454.*
# hung
.*Clamp_1178483.*
# lost results
.*Add_1087636.*
.*Add_2868.*
.*Add_2979.*
.*Add_53543.*

2
inference-engine/tests/functional/plugin/conformance/test_runner/skip_configs/skip_config_template.lst Normal file
View File

@ -0,0 +1,2 @@
# hung
.*AvgPool_1199829.*

5
inference-engine/tests/functional/plugin/conformance/test_runner/src/main.cpp
View File

@ -5,6 +5,7 @@
#include "gtest/gtest.h"
#include "common_test_utils/file_utils.hpp"
#include "functional_test_utils/skip_tests_config.hpp"
#include "shared_test_classes/base/layer_test_utils.hpp"
#include "gflag_config.hpp"
@ -45,6 +46,10 @@ int main(int argc, char* argv[]) {
if (!FLAGS_plugin_lib_name.empty()) {
ConformanceTests::targetPluginName = FLAGS_plugin_lib_name.c_str();
}
if (!FLAGS_skip_config_path.empty()) {
ConformanceTests::disabledTests = FuncTestUtils::SkipTestsConfig::readSkipTestConfigFiles(
CommonTestUtils::splitStringByDelimiter(FLAGS_skip_config_path));
}
::testing::InitGoogleTest(&argc, argv);
::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment);

4
inference-engine/tests/functional/plugin/conformance/test_runner/src/read_ir/read_ir.cpp
View File

@ -10,9 +10,11 @@ namespace ConformanceTests {
using namespace LayerTestsDefinitions;
const char* targetDevice = "";
std::vector<std::string> IRFolderPaths = {};
const char* targetPluginName = "";
std::vector<std::string> IRFolderPaths = {};
std::vector<std::string> disabledTests = {};
namespace {
INSTANTIATE_TEST_CASE_P(conformance,
ReadIRTest,

90
inference-engine/tests/functional/plugin/conformance/test_runner/src/skip_tests_config.cpp
View File

@ -12,93 +12,5 @@
#include "functional_test_utils/skip_tests_config.hpp"
std::vector<std::string> disabledTestPatterns() {
std::vector<std::string> skippedTestsRegExp;
std::string targetDevice = ConformanceTests::targetDevice;
if (targetDevice.find(CommonTestUtils::DEVICE_MYRIAD) != std::string::npos) {
std::vector<std::string> myriadSkips{
// TODO: Issue: 53061
R"(.*ReduceMean_334003.*)", R"(.*ReduceMean_334702.*)", R"(.*Select_1172002.*)", R"(.*Select_1178458.*)",
R"(.*Divide_567867.*)", R"(.*Divide_1377278.*)", R"(.*Divide_1030260.*)", R"(.*Pad_274459.*)", R"(.*Pad_276201.*)",
R"(.*Pad_276193.*)", R"(.*Add_1070296.*)", R"(.*Add_284191.*)", R"(.*Add_1087636.*)", R"(.*VariadicSplit_614428.*)",
R"(.*VariadicSplit_642662.*)", R"(.*VariadicSplit_412430.*)", R"(.*VariadicSplit_475191.*)", R"(.*VariadicSplit_475203.*)",
R"(.*VariadicSplit_279359.*)", R"(.*VariadicSplit_31490.*)", R"(.*VariadicSplit_1444989.*)", R"(.*VariadicSplit_31499.*)",
R"(.*VariadicSplit_279373.*)", R"(.*VariadicSplit_279345.*)", R"(.*VariadicSplit_31508.*)", R"(.*VariadicSplit_535004.*)",
R"(.*VariadicSplit_148861.*)", R"(.*VariadicSplit_475199.*)", R"(.*VariadicSplit_412439.*)", R"(.*VariadicSplit_412448.*)",
R"(.*VariadicSplit_475175.*)", R"(.*VariadicSplit_427269.*)", R"(.*VariadicSplit_1070300.*)", R"(.*VariadicSplit_478455.*)",
R"(.*VariadicSplit_416024.*)", R"(.*VariadicSplit_478469.*)", R"(.*VariadicSplit_416006.*)", R"(.*VariadicSplit_35108.*)",
R"(.*VariadicSplit_478473.*)", R"(.*VariadicSplit_416015.*)", R"(.*VariadicSplit_280446.*)", R"(.*VariadicSplit_6686.*)",
R"(.*VariadicSplit_1087640.*)", R"(.*VariadicSplit_1443292.*)", R"(.*VariadicSplit_35099.*)", R"(.*VariadicSplit_280460.*)",
R"(.*VariadicSplit_35090.*)", R"(.*VariadicSplit_430827.*)", R"(.*VariadicSplit_478461.*)", R"(.*VariadicSplit_280432.*)",
R"(.*VariadicSplit_539066.*)", R"(.*VariadicSplit_149040.*)", R"(.*VariadicSplit_614761.*)", R"(.*VariadicSplit_629649.*)",
R"(.*Unsqueeze_65214.*)", R"(.*Unsqueeze_4838.*)", R"(.*Unsqueeze_6661.*)", R"(.*Multiply_65211.*)", R"(.*Multiply_4833.*)",
R"(.*Multiply_48331.*)", R"(.*Multiply_6656.*)", R"(.*Subtract_534968.*)", R"(.*Subtract_567907.*)", R"(.*Gather_21153.*)",
R"(.*Gather_302464.*)", R"(.*Gather_588814.*)", R"(.*Gather_588810.*)", R"(.*Gather_952843.*)", R"(.*Gather_588818.*)",
R"(.*Gather_588871.*)", R"(.*Gather_48276.*)", R"(.*Gather_1652589.*)", R"(.*Gather_535000.*)", R"(.*Gather_360953.*)",
R"(.*Gather_588867.*)", R"(.*Gather_48283.*)", R"(.*Gather_1311696.*)", R"(.*Gather_588796.*)", R"(.*Gather_4847.*)",
R"(.*Gather_4803.*)", R"(.*Gather_284209.*)", R"(.*Gather_4869.*)", R"(.*Gather_97426.*)", R"(.*Gather_360963.*)", R"(.*Gather_1570871.*)",
R"(.*Gather_513092.*)", R"(.*Gather_629616.*)", R"(.*Gather_48287.*)", R"(.*Gather_284215.*)", R"(.*Gather_48335.*)", R"(.*Gather_1444.*)",
R"(.*Gather_4810.*)", R"(.*Gather_1395.*)", R"(.*Gather_1653002.*)", R"(.*Gather_53520.*)", R"(.*Gather_567864.*)", R"(.*Gather_362776.*)",
R"(.*Gather_567856.*)", R"(.*Gather_567919.*)", R"(.*Gather_567830.*)", R"(.*Gather_567923.*)", R"(.*Gather_567860.*)", R"(.*Gather_539062.*)",
R"(.*Gather_1304430.*)", R"(.*Gather_938915.*)", R"(.*Gather_53527.*)", R"(.*Convert_284211.*)", R"(.*Convert_4830.*)", R"(.*Convert_567844.*)",
R"(.*Convert_6653.*)", R"(.*FakeQuantize_738613.*)", R"(.*FakeQuantize_738639.*)", R"(.*Split_65164.*)", R"(.*Split_274446.*)",
R"(.*Split_361038.*)", R"(.*Split_330570.*)", R"(.*Split_276188.*)", R"(.*Split_68625.*)", R"(.*Split_333279.*)", R"(.*Split_362843.*)",
R"(.*Split_68647.*)", R"(.*Floor_6658.*)", R"(.*TopK_1172093.*)", R"(.*TopK_1172038.*)", R"(.*TopK_1178526.*)", R"(.*TopK_515250.*)",
R"(.*TopK_1178477.*)", R"(.*Reshape_1105704.*)", R"(.*Reshape_1118646.*)", R"(.*Reshape_588858.*)", R"(.*Reshape_567910.*)",
// TODO: Crashes: Should be handled
R"(.*CTCGreedyDecoderSeqLen_271798.*)", R"(.*MVN_276196.*)", R"(.*MVN_274454.*)",
// hung
R"(.*Clamp_1178483.*)",
// lost results
R"(.*Add_1087636.*)", R"(.*Add_2868.*)", R"(.*Add_2979.*)", R"(.*Add_53543.*)",
};
skippedTestsRegExp.insert(skippedTestsRegExp.end(),
std::make_move_iterator(myriadSkips.begin()),
std::make_move_iterator(myriadSkips.end()));
}
if (targetDevice.find(CommonTestUtils::DEVICE_GPU) != std::string::npos) {
std::vector<std::string> gpuSkips{
// TODO: Issue: 53062
R"(.*TensorIterator_1195103.*)", R"(.*TensorIterator_1195129.*)", R"(.*TensorIterator_1653010.*)", R"(.*TensorIterator_199835.*)",
R"(.*TensorIterator_303641.*)", R"(.*TensorIterator_303667.*)", R"(.*TensorIterator_337260.*)", R"(.*TensorIterator_362846.*)",
R"(.*TensorIterator_362869.*)", R"(.*TensorIterator_362906.*)", R"(.*TensorIterator_365496.*)", R"(.*TensorIterator_365522.*)",
R"(.*TensorIterator_365556.*)", R"(.*TensorIterator_365579.*)", R"(.*TensorIterator_616544.*)", R"(.*TensorIterator_616570.*)",
R"(.*TensorIterator_864978.*)", R"(.*TensorIterator_865004.*)", R"(.*TensorIterator_865030.*)", R"(.*TensorIterator_865056.*)",
R"(.*TensorIterator_865082.*)", R"(.*TensorIterator_865108.*)", R"(.*TensorIterator_972832.*)", R"(.*TensorIterator_972858.*)",
R"(.*TensorIterator_972884.*)", R"(.*TensorIterator_972910.*)", R"(.*TensorIterator_972936.*)", R"(.*TensorIterator_972962.*)",
R"(.*TensorIterator_972988.*)", R"(.*TensorIterator_973014.*)", R"(.*TensorIterator_1194007.*)", R"(.*TensorIterator_1194033.*)",
R"(.*TensorIterator_302500.*)", R"(.*TensorIterator_337024.*)", R"(.*TensorIterator_361044.*)", R"(.*TensorIterator_361067.*)",
R"(.*TensorIterator_361108.*)", R"(.*TensorIterator_361131.*)", R"(.*TensorIterator_364171.*)", R"(.*TensorIterator_364197.*)",
R"(.*TensorIterator_364231.*)", R"(.*TensorIterator_364254.*)", R"(.*TensorIterator_615626.*)", R"(.*TensorIterator_615652.*)",
R"(.*TensorIterator_865760.*)", R"(.*TensorIterator_865812.*)", R"(.*TensorIterator_865838.*)", R"(.*TensorIterator_865864.*)",
R"(.*TensorIterator_973781.*)", R"(.*TensorIterator_973807.*)",
// Hung:
R"(.*AvgPool_1199829.*)", R"(.*AvgPool_1201153.*)", R"(.*GroupConvolution_330567.*)", R"(.*ROIPooling_1199827.*)",
R"(.*MaxPool_43108.*)",
};
skippedTestsRegExp.insert(skippedTestsRegExp.end(),
std::make_move_iterator(gpuSkips.begin()),
std::make_move_iterator(gpuSkips.end()));
}
if (targetDevice.find(CommonTestUtils::DEVICE_CPU) != std::string::npos) {
std::vector<std::string> cpuSkips{
// Hung:
R"(.*AvgPool_1199829.*)", R"(.*AvgPool_1201153.*)", R"(.*ROIPooling_1199827.*)",
};
skippedTestsRegExp.insert(skippedTestsRegExp.end(),
std::make_move_iterator(cpuSkips.begin()),
std::make_move_iterator(cpuSkips.end()));
}
if (targetDevice.find(CommonTestUtils::DEVICE_GNA) != std::string::npos) {
std::vector<std::string> gnaSkips{
// Lost results
R"(.*Add_1087636.*)", R"(.*Add_2868.*)", R"(.*Add_2979.*)", R"(.*Add_53543.*)",
// hung
R"(.*Concat_535028.*)", R"(.*Concat_377139.*)", R"(.*Concat_379481.*)", R"(.*Concat_539044.*)", R"(.*Concat_539074.*)",
R"(.*Concat_534956.*)",
};
skippedTestsRegExp.insert(skippedTestsRegExp.end(),
std::make_move_iterator(gnaSkips.begin()),
std::make_move_iterator(gnaSkips.end()));
}
return skippedTestsRegExp;
return ConformanceTests::disabledTests;
}

3
inference-engine/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/skip_tests_config.hpp
View File

@ -7,8 +7,10 @@
#include <vector>
#include <string>
#include <regex>
#include <gtest/gtest.h>
std::vector<std::string> disabledTestPatterns();
namespace FuncTestUtils {
@ -17,6 +19,7 @@ namespace SkipTestsConfig {
extern bool disable_tests_skipping;
bool currentTestIsDisabled();
std::vector<std::string> readSkipTestConfigFiles(const std::vector<std::string>& filePaths);
} // namespace SkipTestsConfig
} // namespace FuncTestUtils

28
inference-engine/tests/ie_test_utils/functional_test_utils/src/skip_tests_config.cpp
View File

@ -2,6 +2,10 @@
// SPDX-License-Identifier: Apache-2.0
//
#include <iostream>
#include <fstream>
#include "common_test_utils/file_utils.hpp"
#include "functional_test_utils/skip_tests_config.hpp"
namespace FuncTestUtils {
@ -21,5 +25,29 @@ bool currentTestIsDisabled() {
return skip_test && !disable_tests_skipping;
}
std::vector<std::string> readSkipTestConfigFiles(const std::vector<std::string>& filePaths) {
std::vector<std::string> res;
for (const auto& filePath : filePaths) {
if (!CommonTestUtils::fileExists(filePath)) {
std::string msg = "Input directory (" + filePath + ") doesn't not exist!";
throw std::runtime_error(msg);
}
std::ifstream file(filePath);
if (file.is_open()) {
std::string buffer;
while (getline(file, buffer)) {
if (buffer.find("#") == std::string::npos && !buffer.empty()) {
res.emplace_back(buffer);
}
}
} else {
std::string msg = "Error in opening file: " + filePath;
throw std::runtime_error(msg);
}
file.close();
}
return res;
}
} // namespace SkipTestsConfig
} // namespace FuncTestUtils