IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Xuejun/behavior tests core threading (#21775)

Browse Source 
* [CPU Func Test] upgrade preprocessing related tests to 2.0

Signed-off-by: Zhai, Xuejun <xuejun.zhai@intel.com>

* [CPU Func Test] upgrade version related tests to 2.0

Signed-off-by: Zhai, Xuejun <xuejun.zhai@intel.com>

* [CPU Func Test] upgrade core threading related tests to 2.0

Signed-off-by: Zhai, Xuejun <xuejun.zhai@intel.com>

* Revert "[CPU Func Test] upgrade preprocessing related tests to 2.0"

This reverts commit abc27eabd2.

---------

Signed-off-by: Zhai, Xuejun <xuejun.zhai@intel.com>
This commit is contained in:
Xuejun Zhai authored and GitHub committed 2023-12-21 11:04:06 +04:00
1 parent 56df9bc75e
commit ca5bf95e26
5 files changed
+376 -29
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_threading_tests.cpp
+23 -7
View File
@@ -2,16 +2,32 @@
// SPDX-License-Identifier: Apache-2.0
//
#include <behavior/ov_plugin/core_threading.hpp>
#include "behavior/ov_plugin/core_threading.hpp"
namespace {
const Params params[] = {
std::tuple<Device, Config>{ov::test::utils::DEVICE_CPU, {{ov::enable_profiling(true)}}},
std::tuple<Device, Config>{ov::test::utils::DEVICE_HETERO, {{"TARGET_FALLBACK", ov::test::utils::DEVICE_CPU}}},
};
const Params paramsStreams[] = {
std::tuple<Device, Config>{ ov::test::utils::DEVICE_CPU, {{ov::num_streams(ov::streams::AUTO)}}},
std::tuple<Device, Config>{ov::test::utils::DEVICE_CPU, {{ov::num_streams(ov::streams::AUTO)}}},
};
} // namespace
INSTANTIATE_TEST_SUITE_P(CPU_Streams, CoreThreadingTestsWithCacheEnabled,
testing::Combine(testing::ValuesIn(paramsStreams),
testing::Values(20),
testing::Values(10)),
CoreThreadingTestsWithCacheEnabled::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(CPU, CoreThreadingTest, testing::ValuesIn(params), CoreThreadingTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(CPU,
CoreThreadingTestsWithIter,
testing::Combine(testing::ValuesIn(params), testing::Values(4), testing::Values(50)),
CoreThreadingTestsWithIter::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(CPU_Streams,
CoreThreadingTestsWithCacheEnabled,
testing::Combine(testing::ValuesIn(paramsStreams), testing::Values(20), testing::Values(10)),
CoreThreadingTestsWithCacheEnabled::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(CPU_Streams,
CoreThreadingTestsWithIter,
testing::Combine(testing::ValuesIn(paramsStreams), testing::Values(4), testing::Values(50)),
CoreThreadingTestsWithIter::getTestCaseName);
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/version.cpp
+23
View File
@@ -0,0 +1,23 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/ov_plugin/version.hpp"
namespace ov {
namespace test {
namespace behavior {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
VersionTests,
::testing::Values(ov::test::utils::DEVICE_CPU),
VersionTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests,
VersionTests,
::testing::Values(ov::test::utils::DEVICE_HETERO),
VersionTests::getTestCaseName);
} // namespace behavior
} // namespace test
} // namespace ov
src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp
+1 -1
View File
@@ -178,7 +178,7 @@ std::vector<std::string> disabledTestPatterns() {
// Old API cannot deallocate tensor
R"(.*InferRequestIOBBlobTest.*canProcessDeallocatedOutputBlobAfterGetAndSetBlob.*)",
// Plugin version was changed to ov::Version
R"(.*VersionTest.*pluginCurrentVersionIsCorrect.*)",
R"(.*VersionTest.pluginCurrentVersionIsCorrect.*)",
// Issue: 113703, 114763
R"(.*smoke_If/SimpleIfTest.*Cond=0.*)",
// Issue: 114765
src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_threading.hpp
+283 -21
View File
@@ -3,14 +3,17 @@
//
#pragma once
#include "openvino/runtime/properties.hpp"
#include <gtest/gtest.h>
#include <thread>
#include <atomic>
#include <mutex>
#include <chrono>
#include <functional_test_utils/skip_tests_config.hpp>
#include "base/ov_behavior_test_utils.hpp"
#include "common_test_utils/ov_tensor_utils.hpp"
#include "common_test_utils/subgraph_builders/2_input_subtract.hpp"
#include "common_test_utils/subgraph_builders/multi_single_conv.hpp"
#include "common_test_utils/subgraph_builders/single_conv.hpp"
#include "common_test_utils/subgraph_builders/split_conv_concat.hpp"
#include "common_test_utils/subgraph_builders/split_multi_conv_concat.hpp"
#include "functional_test_utils/skip_tests_config.hpp"
#include "openvino/runtime/properties.hpp"
using Device = std::string;
using Config = ov::AnyMap;
@@ -19,28 +22,28 @@ using Params = std::tuple<Device, Config>;
class CoreThreadingTestsBase {
public:
static void runParallel(std::function<void(void)> func,
const unsigned int iterations = 100,
const unsigned int threadsNum = 8) {
const unsigned int iterations = 100,
const unsigned int threadsNum = 8) {
std::vector<std::thread> threads(threadsNum);
for (auto & thread : threads) {
thread = std::thread([&](){
for (auto& thread : threads) {
thread = std::thread([&]() {
for (unsigned int i = 0; i < iterations; ++i) {
func();
}
});
}
for (auto & thread : threads) {
for (auto& thread : threads) {
if (thread.joinable())
thread.join();
}
}
void safePluginUnregister(ov::Core & core, const std::string& deviceName) {
void safePluginUnload(ov::Core& core, const std::string& deviceName) {
try {
core.unload_plugin(deviceName);
} catch (const ov::Exception & ex) {
} catch (const ov::Exception& ex) {
// if several threads unload plugin at once, the first thread does this
// while all others will throw an exception that plugin is not registered
ASSERT_STR_CONTAINS(ex.what(), "name is not registered in the");
@@ -60,8 +63,8 @@ using Iterations = unsigned int;
using CoreThreadingParams = std::tuple<Params, Threads, Iterations>;
class CoreThreadingTestsWithCacheEnabled : public testing::WithParamInterface<CoreThreadingParams>,
public ov::test::behavior::OVPluginTestBase,
public CoreThreadingTestsBase {
public ov::test::behavior::OVPluginTestBase,
public CoreThreadingTestsBase {
public:
void SetUp() override {
SKIP_IF_CURRENT_TEST_IS_DISABLED();
@@ -71,8 +74,7 @@ public:
auto hash = std::hash<std::string>()(::testing::UnitTest::GetInstance()->current_test_info()->name());
std::stringstream ss;
ss << std::this_thread::get_id();
cache_path = "threading_test" + std::to_string(hash) + "_"
+ ss.str() + "_" + GetTimestamp() + "_cache";
cache_path = "threading_test" + std::to_string(hash) + "_" + ss.str() + "_" + GetTimestamp() + "_cache";
}
void TearDown() override {
@@ -100,7 +102,6 @@ public:
return result.str();
}
protected:
unsigned int numIterations;
unsigned int numThreads;
@@ -288,8 +289,269 @@ TEST_P(CoreThreadingTestsWithCacheEnabled, smoke_compilemodel_cache_enabled) {
SetupModel();
core.set_property(target_device, config);
core.set_property(ov::cache_dir(cache_path));
runParallel([&] () {
(void)core.compile_model(model, target_device);
}, numIterations, numThreads);
runParallel(
[&]() {
(void)core.compile_model(model, target_device);
},
numIterations,
numThreads);
core.set_property(ov::cache_dir(""));
}
class CoreThreadingTest : public testing::WithParamInterface<Params>,
public ov::test::behavior::OVPluginTestBase,
public CoreThreadingTestsBase {
public:
void SetUp() override {
std::tie(target_device, config) = GetParam();
APIBaseTest::SetUp();
SKIP_IF_CURRENT_TEST_IS_DISABLED();
}
static std::string getTestCaseName(testing::TestParamInfo<Params> obj) {
std::string deviceName;
Config config;
std::tie(deviceName, config) = obj.param;
std::replace(deviceName.begin(), deviceName.end(), ':', '.');
char separator('_');
std::ostringstream result;
result << "targetDevice=" << deviceName << separator;
result << "config=";
for (auto& confItem : config) {
result << confItem.first << "=" << confItem.second.as<std::string>() << separator;
}
return result.str();
}
};
// tested function: get_versions, unload_plugin
TEST_P(CoreThreadingTest, smoke_GetVersions) {
ov::Core core;
runParallel([&]() {
auto versions = core.get_versions(target_device);
ASSERT_LE(1u, versions.size());
safePluginUnload(core, target_device);
});
}
// tested function: set_property for already created plugins
TEST_P(CoreThreadingTest, smoke_SetProperty_PluginExists) {
ov::Core core;
core.set_property(config);
auto versions = core.get_versions(target_device);
runParallel(
[&]() {
core.set_property(config);
},
10000);
}
// tested function: get_property, unload_plugin
TEST_P(CoreThreadingTest, smoke_GetProperty_PluginExists) {
ov::Core core;
std::string configKey = config.begin()->first;
core.set_property(config);
runParallel([&]() {
core.get_property(target_device, configKey);
safePluginUnload(core, target_device);
});
}
// tested function: query_model
TEST_P(CoreThreadingTest, smoke_QueryModel) {
ov::Core core;
auto model = ov::test::utils::make_2_input_subtract();
core.set_property(target_device, config);
auto refResult = core.query_model(model, target_device);
runParallel(
[&]() {
const auto result = core.query_model(model, target_device);
safePluginUnload(core, target_device);
// compare QueryNetworkResult with reference
for (auto&& r : refResult) {
ASSERT_NE(result.end(), result.find(r.first));
}
for (auto&& r : result) {
ASSERT_NE(refResult.end(), refResult.find(r.first));
}
},
3000);
}
class CoreThreadingTestsWithIter : public testing::WithParamInterface<CoreThreadingParams>,
public ov::test::behavior::OVPluginTestBase,
public CoreThreadingTestsBase {
public:
void SetUp() override {
SKIP_IF_CURRENT_TEST_IS_DISABLED();
std::tie(target_device, config) = std::get<0>(GetParam());
numThreads = std::get<1>(GetParam());
numIterations = std::get<2>(GetParam());
}
static std::string getTestCaseName(testing::TestParamInfo<CoreThreadingParams> obj) {
unsigned int numThreads, numIterations;
std::string deviceName;
Config config;
std::tie(deviceName, config) = std::get<0>(obj.param);
std::replace(deviceName.begin(), deviceName.end(), ':', '.');
numThreads = std::get<1>(obj.param);
numIterations = std::get<2>(obj.param);
char separator('_');
std::ostringstream result;
result << "targetDevice=" << deviceName << separator;
result << "config=";
for (auto& confItem : config) {
result << confItem.first << "=" << confItem.second.as<std::string>() << separator;
}
result << "numThreads=" << numThreads << separator;
result << "numIter=" << numIterations;
return result.str();
}
protected:
unsigned int numIterations;
unsigned int numThreads;
std::vector<std::shared_ptr<ov::Model>> models;
void SetupNetworks() {
models.emplace_back(ov::test::utils::make_2_input_subtract());
models.emplace_back(ov::test::utils::make_multi_single_conv());
models.emplace_back(ov::test::utils::make_single_conv());
models.emplace_back(ov::test::utils::make_split_conv_concat());
models.emplace_back(ov::test::utils::make_split_multi_conv_concat());
}
};
// tested function: compile_model
TEST_P(CoreThreadingTestsWithIter, smoke_CompileModel) {
ov::Core core;
std::atomic<unsigned int> counter{0u};
SetupNetworks();
core.set_property(target_device, config);
runParallel(
[&]() {
auto value = counter++;
(void)core.compile_model(models[value % models.size()], target_device);
},
numIterations,
numThreads);
}
// tested function: single Core compile_model accuracy
TEST_P(CoreThreadingTestsWithIter, smoke_CompileModel_Accuracy_SingleCore) {
ov::Core core;
std::atomic<unsigned int> counter{0u};
SetupNetworks();
core.set_property(target_device, config);
runParallel(
[&]() {
auto value = counter++;
auto model = models[value % models.size()];
std::map<ov::Output<ov::Node>, ov::Tensor> inputs;
for (const auto& input : model->inputs()) {
auto tensor = ov::test::utils::create_and_fill_tensor_normal_distribution(input.get_element_type(),
input.get_shape(),
0.0f,
0.2f,
7235346);
inputs.insert({input, tensor});
}
auto getOutputBlob = [&](ov::Core& core) {
auto compiled_model = core.compile_model(model, target_device);
auto req = compiled_model.create_infer_request();
for (const auto& input : inputs) {
req.set_tensor(input.first, input.second);
}
auto output_tensor = ov::Tensor(model->output().get_element_type(), model->output().get_shape());
req.set_output_tensor(output_tensor);
req.infer();
return output_tensor;
};
auto outputActual = getOutputBlob(core);
// compare actual value using the same Core
auto outputRef = getOutputBlob(core);
ov::test::utils::compare(outputActual, outputRef);
},
numIterations,
numThreads);
}
// tested function: multi Core compile_model accuracy
TEST_P(CoreThreadingTestsWithIter, smoke_CompileModel_Accuracy_MultipleCores) {
ov::Core core;
std::atomic<unsigned int> counter{0u};
SetupNetworks();
core.set_property(target_device, config);
runParallel(
[&]() {
auto value = counter++;
auto model = models[value % models.size()];
std::map<ov::Output<ov::Node>, ov::Tensor> inputs;
for (const auto& input : model->inputs()) {
auto tensor = ov::test::utils::create_and_fill_tensor_normal_distribution(input.get_element_type(),
input.get_shape(),
0.0f,
0.2f,
7235346);
inputs.insert({input, tensor});
}
auto getOutputBlob = [&](ov::Core& core) {
auto compiled_model = core.compile_model(model, target_device);
auto req = compiled_model.create_infer_request();
for (const auto& input : inputs) {
req.set_tensor(input.first, input.second);
}
auto output_tensor = ov::Tensor(model->output().get_element_type(), model->output().get_shape());
req.set_output_tensor(output_tensor);
req.infer();
return output_tensor;
};
auto outputActual = getOutputBlob(core);
// compare actual value using the second Core
{
ov::Core core2;
core2.set_property(target_device, config);
auto outputRef = getOutputBlob(core2);
ov::test::utils::compare(outputActual, outputRef);
}
},
numIterations,
numThreads);
}
// tested function: set_property, compile_model
TEST_P(CoreThreadingTestsWithIter, smoke_CompileModel_MultipleCores) {
std::atomic<unsigned int> counter{0u};
SetupNetworks();
runParallel(
[&]() {
auto value = counter++;
ov::Core core;
core.set_property(target_device, config);
(void)core.compile_model(models[value % models.size()], target_device);
},
numIterations,
numThreads);
}
src/tests/functional/plugin/shared/include/behavior/ov_plugin/version.hpp
+46
View File
@@ -0,0 +1,46 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "base/behavior_test_utils.hpp"
namespace ov {
namespace test {
namespace behavior {
class VersionTests : public testing::WithParamInterface<std::string>, virtual public OVPluginTestBase {
public:
static std::string getTestCaseName(testing::TestParamInfo<std::string> obj) {
std::string targetDevice;
std::map<std::string, std::string> config;
targetDevice = obj.param;
std::replace(targetDevice.begin(), targetDevice.end(), ':', '_');
std::ostringstream result;
result << "targetDevice=" << targetDevice;
return result.str();
}
void SetUp() override {
target_device = this->GetParam();
SKIP_IF_CURRENT_TEST_IS_DISABLED()
APIBaseTest::SetUp();
}
std::shared_ptr<ov::Core> core = ov::test::utils::PluginCache::get().core();
};
// Load unsupported network type to the Plugin
TEST_P(VersionTests, pluginCurrentVersionIsCorrect) {
if (target_device.find(ov::test::utils::DEVICE_AUTO) == std::string::npos &&
target_device.find(ov::test::utils::DEVICE_MULTI) == std::string::npos &&
target_device.find(ov::test::utils::DEVICE_HETERO) == std::string::npos) {
std::map<std::string, ov::Version> versions = core->get_versions(target_device);
ASSERT_EQ(versions.size(), 1);
ASSERT_EQ(versions.begin()->first, target_device);
auto version = versions.begin()->second;
EXPECT_TRUE(version.buildNumber != nullptr);
EXPECT_TRUE(version.description != nullptr);
}
}
} // namespace behavior
} // namespace test
} // namespace ov