IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[IE TESTS][Conformance] API conformance instances (#7948)

Browse Source 
* [IE TESTS] Add API Conformance

* Inst

* temp

* Small refactoring

* Instantiatians

* ov_plugihn

* Fix some cases

* fix template

* Six oss build

* Update conformance.hpp

* fix

* try to fix

* Apply comments

* Remove extra
This commit is contained in:
Irina Efode 2021-11-18 13:44:44 +03:00 committed by GitHub
parent a84d01cb0d
commit 2468a015dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 1423 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/CMakeLists.txt
View File

@ -10,6 +10,9 @@ addIeTargetTest(
ADDITIONAL_SOURCE_DIRS
${CMAKE_CURRENT_SOURCE_DIR}/src
ADD_CPPLINT
INCLUDES
PRIVATE
"${CMAKE_CURRENT_SOURCE_DIR}/include"
LINK_LIBRARIES
PUBLIC
conformanceShared

86
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/api_conformance_helpers.hpp Normal file
View File

@ -0,0 +1,86 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "conformance.hpp"
#include "common_test_utils/test_constants.hpp"
// TODO: fix namespaces
using namespace ConformanceTests;
namespace ov {
namespace test {
namespace conformance {
inline const std::string getPluginLibNameByDevice(const std::string& deviceName) {
const std::map<std::string, std::string> devices{
{ "AUTO", "AutoPlugin" },
{ "HDDL", "HDDLPlugin" },
{ "VPUX", "VPUXPlugin" },
{ "AUTO", "MultiDevicePlugin" },
{ "CPU", "MKLDNNPlugin" },
{ "GNA", "GNAPlugin" },
{ "GPU", "clDNNPlugin" },
{ "HETERO", "HeteroPlugin" },
{ "MULTI", "MultiDevicePlugin" },
{ "MYRIAD", "myriadPlugin" },
{ "TEMPLATE", "templatePlugin" },
};
if (devices.find(deviceName) == devices.end()) {
throw std::runtime_error("Incorrect device name");
}
return devices.at(deviceName);
}
inline const std::pair<std::string, std::string> generateDefaultMultiConfig() {
return {MULTI_CONFIG_KEY(DEVICE_PRIORITIES), ConformanceTests::targetDevice};
}
inline const std::pair<std::string, std::string> generateDefaultHeteroConfig() {
return { "TARGET_FALLBACK" , ConformanceTests::targetDevice };
}
inline const std::vector<std::map<std::string, std::string>> generateConfigs(const std::string& targetDevice,
const std::vector<std::map<std::string, std::string>>& config = {}) {
std::pair<std::string, std::string> defaultConfig;
if (targetDevice == std::string(CommonTestUtils::DEVICE_MULTI) || targetDevice == std::string(CommonTestUtils::DEVICE_AUTO)) {
defaultConfig = generateDefaultMultiConfig();
} else if (targetDevice == std::string(CommonTestUtils::DEVICE_HETERO)) {
defaultConfig = generateDefaultHeteroConfig();
} else {
throw std::runtime_error("Incorrect target device: " + targetDevice);
}
std::vector<std::map<std::string, std::string>> resultConfig;
if (config.empty()) {
return {{defaultConfig}};
}
for (auto configItem : config) {
configItem.insert(defaultConfig);
resultConfig.push_back(configItem);
}
return resultConfig;
}
inline const std::string generateComplexDeviceName(const std::string& deviceName) {
return deviceName + ":" + ConformanceTests::targetDevice;
}
inline const std::vector<std::string> returnAllPossibleDeviceCombination() {
std::vector<std::string> res{ConformanceTests::targetDevice};
std::vector<std::string> devices{CommonTestUtils::DEVICE_HETERO, CommonTestUtils::DEVICE_AUTO, CommonTestUtils::DEVICE_MULTI};
for (const auto& device : devices) {
res.emplace_back(generateComplexDeviceName(device));
}
return res;
}
const std::vector<std::map<std::string, std::string>> emptyConfig = {
{},
};
} // namespace conformance
} // namespace test
} // namespace ov

40
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/caching/caching_tests.cpp Normal file
View File

@ -0,0 +1,40 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/caching/caching_tests.hpp"
#include <ngraph_ops/nms_ie_internal.hpp>
#include <ngraph_ops/nms_static_shape_ie.hpp>
#include "conformance.hpp"
namespace {
using namespace LayerTestsDefinitions;
using namespace ngraph;
static const std::vector<ov::element::Type> precisionsTemplate = {
ov::element::f64,
ov::element::f32,
ov::element::f16,
ov::element::i64,
ov::element::i32,
ov::element::i16,
ov::element::i8,
ov::element::u64,
ov::element::u32,
ov::element::u16,
ov::element::u8,
ov::element::boolean,
};
static const std::vector<std::size_t> batchSizesTemplate = {
1, 2
};
INSTANTIATE_TEST_SUITE_P(smoke_Behavior_CachingSupportCase, LoadNetworkCacheTestBase,
::testing::Combine(
::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()),
::testing::ValuesIn(precisionsTemplate),
::testing::ValuesIn(batchSizesTemplate),
::testing::Values(ConformanceTests::targetDevice)),
LoadNetworkCacheTestBase::getTestCaseName);
} // namespace

29
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/executable_network/exec_graph_info.cpp Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <common_test_utils/test_constants.hpp>
#include <exec_graph_info.hpp>
#include "behavior/executable_network/exec_graph_info.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace ExecutionGraphTests;
INSTANTIATE_TEST_SUITE_P(smoke_serialization, ExecGraphSerializationTest,
::testing::Values(ConformanceTests::targetDevice),
ExecGraphSerializationTest::getTestCaseName);
const std::vector<InferenceEngine::Precision> execGraphInfoElemTypes = {
InferenceEngine::Precision::FP32
};
INSTANTIATE_TEST_SUITE_P(smoke_NoReshape, ExecGraphUniqueNodeNames,
::testing::Combine(
::testing::ValuesIn(execGraphInfoElemTypes),
::testing::Values(InferenceEngine::SizeVector({1, 2, 5, 5})),
::testing::Values(ConformanceTests::targetDevice)),
ExecGraphUniqueNodeNames::getTestCaseName);
} // namespace

66
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/executable_network/exec_network_base.cpp Normal file
View File

@ -0,0 +1,66 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/executable_network/exec_network_base.hpp"
#include "ie_plugin_config.hpp"
#include "api_conformance_helpers.hpp"
using namespace BehaviorTestsDefinitions;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, ExecutableNetworkBaseTest,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
ExecutableNetworkBaseTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, ExecutableNetworkBaseTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
ExecutableNetworkBaseTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, ExecutableNetworkBaseTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
ExecutableNetworkBaseTest::getTestCaseName);
const std::vector<InferenceEngine::Precision> execNetBaseElemTypes = {
InferenceEngine::Precision::FP32,
InferenceEngine::Precision::U8,
InferenceEngine::Precision::I16,
InferenceEngine::Precision::U16
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, ExecNetSetPrecision,
::testing::Combine(
::testing::ValuesIn(execNetBaseElemTypes),
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
ExecNetSetPrecision::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, ExecNetSetPrecision,
::testing::Combine(
::testing::ValuesIn(execNetBaseElemTypes),
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
ExecNetSetPrecision::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, ExecNetSetPrecision,
::testing::Combine(
::testing::ValuesIn(execNetBaseElemTypes),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
ExecNetSetPrecision::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, ExecNetSetPrecision,
::testing::Combine(
::testing::ValuesIn(execNetBaseElemTypes),
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
ExecNetSetPrecision::getTestCaseName);
} // namespace

74
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/executable_network/get_metric.cpp Normal file
View File

@ -0,0 +1,74 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/executable_network/get_metric.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::conformance;
using namespace BehaviorTestsDefinitions;
using namespace InferenceEngine::PluginConfigParams;
namespace {
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassImportExportTestP, IEClassImportExportTestP,
::testing::Values(generateComplexDeviceName(CommonTestUtils::DEVICE_HETERO)));
//
// Executable Network GetMetric
//
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_ThrowsUnsupported,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
//
// Executable Network GetConfig / SetConfig
//
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassExecutableNetworkGetConfigTest, IEClassExecutableNetworkGetConfigTest,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
::testing::Values(ConformanceTests::targetDevice));
//
// Hetero Executable Network GetMetric
//
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassHeteroExecutableNetworkGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassHeteroExecutableNetworkGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassHeteroExecutableNetworkGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassHeteroExecutableNetworkGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK,
::testing::Values(ConformanceTests::targetDevice));
} // namespace

27
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/infer_request/callback.cpp
View File

@ -3,39 +3,34 @@
//
#include "behavior/infer_request/callback.hpp"
#include "conformance.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace ov::test::conformance;
using namespace BehaviorTestsDefinitions;
using namespace ConformanceTests;
const std::vector<std::map<std::string, std::string>> configsCallback = {
{},
};
const std::vector<std::map<std::string, std::string>> multiConfigsCallback = {
{{MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice}}
};
const std::vector<std::map<std::string, std::string>> autoConfigsCallback = {
{{MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice}}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestCallbackTests,
::testing::Combine(
::testing::Values(targetDevice),
::testing::ValuesIn(configsCallback)),
::testing::ValuesIn(emptyConfig)),
InferRequestCallbackTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestCallbackTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(multiConfigsCallback)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestCallbackTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestCallbackTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(autoConfigsCallback)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestCallbackTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestCallbackTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestCallbackTests::getTestCaseName);
} // namespace

9
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/infer_request/cancellation.cpp
View File

@ -3,19 +3,16 @@
//
#include "behavior/infer_request/cancellation.hpp"
#include "conformance.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace BehaviorTestsDefinitions;
using namespace ConformanceTests;
const std::vector<std::map<std::string, std::string>> configsCancel = {
{},
};
using namespace ov::test::conformance;
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestCancellationTests,
::testing::Combine(
::testing::Values(targetDevice),
::testing::ValuesIn(configsCancel)),
::testing::ValuesIn(emptyConfig)),
InferRequestCancellationTests::getTestCaseName);
} // namespace

27
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/infer_request/io_blob.cpp
View File

@ -6,39 +6,34 @@
#include "behavior/infer_request/io_blob.hpp"
#include "ie_plugin_config.hpp"
#include "conformance.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace ov::test::conformance;
using namespace BehaviorTestsDefinitions;
using namespace ConformanceTests;
const std::vector<std::map<std::string, std::string>> configsIOBlob = {
{},
};
const std::vector<std::map<std::string, std::string>> MulticonfigsIOBlob = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice }}
};
const std::vector<std::map<std::string, std::string>> AutoconfigsIOBlob = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice}}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestIOBBlobTest,
::testing::Combine(
::testing::Values(targetDevice),
::testing::ValuesIn(configsIOBlob)),
::testing::ValuesIn(emptyConfig)),
InferRequestIOBBlobTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestIOBBlobTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(MulticonfigsIOBlob)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestIOBBlobTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestIOBBlobTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(AutoconfigsIOBlob)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestIOBBlobTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestIOBBlobTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestIOBBlobTest::getTestCaseName);
} // namespace

28
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/infer_request/multitheading.cpp
View File

@ -7,41 +7,35 @@
#include "behavior/infer_request/multithreading.hpp"
#include "ie_plugin_config.hpp"
#include "conformance.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace ov::test::conformance;
using namespace ConformanceTests;
using namespace BehaviorTestsDefinitions;
const std::vector<std::map<std::string, std::string>> configsMultithreading = {
{},
};
const std::vector<std::map<std::string, std::string>> MulticonfigsMultithreading = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice }}
};
const std::vector<std::map<std::string, std::string>> AutoconfigsMultithreading = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice}}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestMultithreadingTests,
::testing::Combine(
::testing::Values(targetDevice),
::testing::ValuesIn(configsMultithreading)),
::testing::ValuesIn(emptyConfig)),
InferRequestMultithreadingTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestMultithreadingTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(MulticonfigsMultithreading)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestMultithreadingTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestMultithreadingTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(AutoconfigsMultithreading)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestMultithreadingTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestMultithreadingTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestMultithreadingTests::getTestCaseName);
} // namespace

28
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/infer_request/perf_counters.cpp
View File

@ -3,41 +3,35 @@
//
#include "behavior/infer_request/perf_counters.hpp"
#include "conformance.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace ov::test::conformance;
using namespace ConformanceTests;
using namespace BehaviorTestsDefinitions;
const std::vector<std::map<std::string, std::string>> configsPerfCounters = {
{}
};
const std::vector<std::map<std::string, std::string>> MulticonfigsPerfCounters = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice }}
};
const std::vector<std::map<std::string, std::string>> AutoconfigsPerfCounters = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice }}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(targetDevice),
::testing::ValuesIn(configsPerfCounters)),
::testing::ValuesIn(emptyConfig)),
InferRequestPerfCountersTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(MulticonfigsPerfCounters)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestPerfCountersTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(AutoconfigsPerfCounters)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestPerfCountersTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestPerfCountersTest::getTestCaseName);

23
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/infer_request/set_blob_by_type.cpp
View File

@ -4,14 +4,14 @@
#include "behavior/infer_request/set_blob_by_type.hpp"
#include "common_test_utils/test_constants.hpp"
#include "conformance.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace ov::test::conformance;
using namespace ConformanceTests;
using namespace BehaviorTestsDefinitions;
const std::vector<FuncTestUtils::BlobType> BlobTypes = {
const std::vector<FuncTestUtils::BlobType> setBlobTypes = {
FuncTestUtils::BlobType::Compound,
FuncTestUtils::BlobType::Batched,
FuncTestUtils::BlobType::Memory,
@ -21,32 +21,29 @@ const std::vector<FuncTestUtils::BlobType> BlobTypes = {
};
const std::map<std::string, std::string> ConfigBlobType{}; //nothing special
const std::map<std::string, std::string> autoConfigBlobType{};
const std::map<std::string, std::string> multiConfigBlobType{{MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice}};
const std::map<std::string, std::string> heteroConfigBlobType{{"TARGET_FALLBACK", targetDevice}};
INSTANTIATE_TEST_SUITE_P(smoke_Behavior, InferRequestSetBlobByType,
::testing::Combine(::testing::ValuesIn(BlobTypes),
::testing::Combine(::testing::ValuesIn(setBlobTypes),
::testing::Values(targetDevice),
::testing::Values(ConfigBlobType)),
InferRequestSetBlobByType::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Behavior_Multi, InferRequestSetBlobByType,
::testing::Combine(::testing::ValuesIn(BlobTypes),
::testing::Combine(::testing::ValuesIn(setBlobTypes),
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::Values(multiConfigBlobType)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestSetBlobByType::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Behavior_Auto, InferRequestSetBlobByType,
::testing::Combine(::testing::ValuesIn(BlobTypes),
::testing::Combine(::testing::ValuesIn(setBlobTypes),
::testing::Values(CommonTestUtils::DEVICE_AUTO + std::string(":") + targetDevice),
::testing::Values(autoConfigBlobType)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestSetBlobByType::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Behavior_Hetero, InferRequestSetBlobByType,
::testing::Combine(::testing::ValuesIn(BlobTypes),
::testing::Combine(::testing::ValuesIn(setBlobTypes),
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::Values(heteroConfigBlobType)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestSetBlobByType::getTestCaseName);
} // namespace

28
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/infer_request/wait.cpp
View File

@ -6,41 +6,35 @@
#include "behavior/infer_request/wait.hpp"
#include "ie_plugin_config.hpp"
#include "conformance.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace ov::test::conformance;
using namespace ConformanceTests;
using namespace BehaviorTestsDefinitions;
const std::vector<std::map<std::string, std::string>> configsWait = {
{},
};
const std::vector<std::map<std::string, std::string>> MulticonfigsWait = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES) , targetDevice}}
};
const std::vector<std::map<std::string, std::string>> AutoconfigsWait = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES) , targetDevice}}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestWaitTests,
::testing::Combine(
::testing::Values(targetDevice),
::testing::ValuesIn(configsWait)),
::testing::ValuesIn(emptyConfig)),
InferRequestWaitTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestWaitTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(MulticonfigsWait)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestWaitTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestWaitTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(AutoconfigsWait)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestWaitTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestWaitTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestWaitTests::getTestCaseName);
} // namespace

58
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_graph_info.cpp Normal file
View File

@ -0,0 +1,58 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/ov_executable_network/exec_graph_info.hpp"
#include "api_conformance_helpers.hpp"
#include "ie_plugin_config.hpp"
#include <common_test_utils/test_constants.hpp>
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
const std::vector<ov::element::Type_t> ovExecGraphInfoElemTypes = {
ov::element::i8,
ov::element::i16,
ov::element::i32,
ov::element::i64,
ov::element::u8,
ov::element::u16,
ov::element::u32,
ov::element::u64,
ov::element::f16,
ov::element::f32,
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
OVExecGraphImportExportTest,
::testing::Combine(
::testing::ValuesIn(ovExecGraphInfoElemTypes),
::testing::Values(CommonTestUtils::DEVICE_CPU),
::testing::ValuesIn(emptyConfig)),
OVExecGraphImportExportTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests,
OVExecGraphImportExportTest,
::testing::Combine(
::testing::ValuesIn(ovExecGraphInfoElemTypes),
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVExecGraphImportExportTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests,
OVExecGraphImportExportTest,
::testing::Combine(
::testing::ValuesIn(ovExecGraphInfoElemTypes),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVExecGraphImportExportTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests,
OVExecGraphImportExportTest,
::testing::Combine(::testing::ValuesIn(ovExecGraphInfoElemTypes),
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVExecGraphImportExportTest::getTestCaseName);
} // namespace

36
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_network_base.cpp Normal file
View File

@ -0,0 +1,36 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/ov_executable_network/exec_network_base.hpp"
#include "ie_plugin_config.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVExecutableNetworkBaseTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_CPU),
::testing::ValuesIn(emptyConfig)),
OVExecutableNetworkBaseTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVExecutableNetworkBaseTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVExecutableNetworkBaseTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVExecutableNetworkBaseTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVExecutableNetworkBaseTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVExecutableNetworkBaseTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVExecutableNetworkBaseTest::getTestCaseName);
} // namespace

83
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/get_metric.cpp Normal file
View File

@ -0,0 +1,83 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/ov_executable_network/get_metric.hpp"
#include "openvino/runtime/core.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
using namespace InferenceEngine::PluginConfigParams;
namespace {
//
// IE Class Common tests with <pluginName, deviceName params>
//
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassImportExportTestP, OVClassImportExportTestP,
::testing::Values(generateComplexDeviceName(CommonTestUtils::DEVICE_HETERO)));
//
// Executable Network GetMetric
//
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassExecutableNetworkGetMetricTest, OVClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassExecutableNetworkGetMetricTest, OVClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassExecutableNetworkGetMetricTest, OVClassExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassExecutableNetworkGetMetricTest, OVClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassExecutableNetworkGetMetricTest, OVClassExecutableNetworkGetMetricTest_ThrowsUnsupported,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
//
// Executable Network GetConfig / SetConfig
//
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassExecutableNetworkGetConfigTest, OVClassExecutableNetworkGetConfigTest,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassExecutableNetworkSetConfigTest, OVClassExecutableNetworkSetConfigTest,
::testing::Values(ConformanceTests::targetDevice));
////
//// Hetero Executable Network GetMetric
////
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassHeteroExecutableNetworkGetMetricTest, OVClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassHeteroExecutableNetworkGetMetricTest, OVClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassHeteroExecutableNetworkGetMetricTest, OVClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassHeteroExecutableNetworkGetMetricTest, OVClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK,
::testing::Values(ConformanceTests::targetDevice));
//////////////////////////////////////////////////////////////////////////////////////////
} // namespace

37
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/callback.cpp Normal file
View File

@ -0,0 +1,37 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <vector>
#include "behavior/ov_infer_request/callback.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestCallbackTests,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestCallbackTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestCallbackTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVInferRequestCallbackTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestCallbackTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVInferRequestCallbackTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestCallbackTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVInferRequestCallbackTests::getTestCaseName);
} // namespace

17
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/cancellation.cpp Normal file
View File

@ -0,0 +1,17 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/ov_infer_request/cancellation.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestCancellationTests,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestCancellationTests::getTestCaseName);
} // namespace

106
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/infer_request_dynamic.cpp Normal file
View File

@ -0,0 +1,106 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <vector>
#include "behavior/ov_infer_request/infer_request_dynamic.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
std::shared_ptr<ngraph::Function> ovGetFunction1() {
const std::vector<size_t> inputShape = {1, 4, 20, 20};
const ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32;
auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
params.front()->set_friendly_name("Param_1");
params.front()->get_output_tensor(0).set_names({"input_tensor"});
auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 4, 1, 1}, std::vector<float>{}, true);
auto add = ngraph::builder::makeEltwise(params[0], in2add, ngraph::helpers::EltwiseTypes::ADD);
auto relu1 = std::make_shared<ngraph::opset1::Relu>(add->output(0));
relu1->get_output_tensor(0).set_names({"relu1"});
auto relu2 = std::make_shared<ngraph::opset1::Relu>(add->output(0));
relu2->get_output_tensor(0).set_names({"relu2"});
ngraph::NodeVector results{relu1, relu2};
return std::make_shared<ngraph::Function>(results, params, "AddTwoOutputEdges");
}
std::shared_ptr<ngraph::Function> ovGetFunction2() {
const std::vector<size_t> inputShape = {1, 4, 20, 20};
const ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32;
auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
params.front()->set_friendly_name("Param_1");
params.front()->get_output_tensor(0).set_names({"input_tensor"});
auto split = ngraph::builder::makeSplit(params[0], ngPrc, 2, 1);
auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 2, 1, 1}, std::vector<float>{}, true);
auto add = ngraph::builder::makeEltwise(split->output(0), in2add, ngraph::helpers::EltwiseTypes::ADD);
auto relu1 = std::make_shared<ngraph::opset1::Relu>(add);
auto in2mult = ngraph::builder::makeConstant(ngPrc, {1, 2, 1, 1}, std::vector<float>{}, true);
auto mult = ngraph::builder::makeEltwise(split->output(1), in2mult, ngraph::helpers::EltwiseTypes::MULTIPLY);
auto relu2 = std::make_shared<ngraph::opset1::Relu>(mult);
auto concat = std::make_shared<ngraph::opset1::Concat>(ngraph::OutputVector{relu1->output(0), relu2->output(0)}, 3);
concat->get_output_tensor(0).set_names({"concat"});
return std::make_shared<ngraph::Function>(concat, params, "SplitAddConcat");
}
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_1, OVInferRequestDynamicTests,
::testing::Combine(
::testing::Values(ovGetFunction1()),
::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
{{1, 4, 20, 20}, {1, 4, 20, 20}},
{{2, 4, 20, 20}, {2, 4, 20, 20}}}),
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestDynamicTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_2, OVInferRequestDynamicTests,
::testing::Combine(
::testing::Values(ovGetFunction2()),
::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
{{1, 4, 20, 20}, {1, 2, 20, 40}},
{{2, 4, 20, 20}, {2, 2, 20, 40}}}),
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestDynamicTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestDynamicTests,
::testing::Combine(
::testing::Values(ovGetFunction2()),
::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
{{1, 4, 20, 20}, {1, 2, 20, 40}},
{{2, 4, 20, 20}, {2, 2, 20, 40}}}),
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVInferRequestDynamicTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestDynamicTests,
::testing::Combine(
::testing::Values(ovGetFunction2()),
::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
{{1, 4, 20, 20}, {1, 2, 20, 40}},
{{2, 4, 20, 20}, {2, 2, 20, 40}}}),
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVInferRequestDynamicTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestDynamicTests,
::testing::Combine(
::testing::Values(ovGetFunction2()),
::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
{{1, 4, 20, 20}, {1, 2, 20, 40}},
{{2, 4, 20, 20}, {2, 2, 20, 40}}}),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVInferRequestDynamicTests::getTestCaseName);
} // namespace

36
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/inference_chaining.cpp Normal file
View File

@ -0,0 +1,36 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/ov_infer_request/inference_chaining.hpp"
#include "common_test_utils/test_constants.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferenceChaining,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferenceChaining::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferenceChaining,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVInferenceChaining::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferenceChaining,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVInferenceChaining::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferenceChaining,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVInferenceChaining::getTestCaseName);
} // namespace

85
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/io_tensor.cpp Normal file
View File

@ -0,0 +1,85 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <vector>
#include "behavior/ov_infer_request/io_tensor.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestIOTensorTest,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestIOTensorTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestIOTensorTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVInferRequestIOTensorTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestIOTensorTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVInferRequestIOTensorTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestIOTensorTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVInferRequestIOTensorTest::getTestCaseName);
std::vector<ov::element::Type> ovIOTensorElemTypes = {
ov::element::boolean,
ov::element::bf16,
ov::element::f16,
ov::element::f32,
ov::element::f64,
ov::element::i4,
ov::element::i8,
ov::element::i16,
ov::element::i32,
ov::element::i64,
ov::element::u1,
ov::element::u4,
ov::element::u8,
ov::element::u16,
ov::element::u32,
ov::element::u64,
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestIOTensorSetPrecisionTest,
::testing::Combine(
::testing::ValuesIn(ovIOTensorElemTypes),
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestIOTensorSetPrecisionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestIOTensorSetPrecisionTest,
::testing::Combine(
::testing::ValuesIn(ovIOTensorElemTypes),
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVInferRequestIOTensorSetPrecisionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestIOTensorSetPrecisionTest,
::testing::Combine(
::testing::ValuesIn(ovIOTensorElemTypes),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVInferRequestIOTensorSetPrecisionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestIOTensorSetPrecisionTest,
::testing::Combine(
::testing::ValuesIn(ovIOTensorElemTypes),
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVInferRequestIOTensorSetPrecisionTest::getTestCaseName);
} // namespace

39
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/multithreading.cpp Normal file
View File

@ -0,0 +1,39 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <vector>
#include "behavior/ov_infer_request/multithreading.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestMultithreadingTests,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestMultithreadingTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestMultithreadingTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVInferRequestMultithreadingTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestMultithreadingTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVInferRequestMultithreadingTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestMultithreadingTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVInferRequestMultithreadingTests::getTestCaseName);
} // namespace

35
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/perf_counters.cpp Normal file
View File

@ -0,0 +1,35 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/ov_infer_request/perf_counters.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestPerfCountersTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVInferRequestPerfCountersTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVInferRequestPerfCountersTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVInferRequestPerfCountersTest::getTestCaseName);
} // namespace

38
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/wait.cpp Normal file
View File

@ -0,0 +1,38 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <vector>
#include "behavior/ov_infer_request/wait.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestWaitTests,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(emptyConfig)),
OVInferRequestWaitTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestWaitTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
OVInferRequestWaitTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestWaitTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
OVInferRequestWaitTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestWaitTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
OVInferRequestWaitTests::getTestCaseName);
} // namespace

94
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/core_integration.cpp Normal file
View File

@ -0,0 +1,94 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/ov_plugin/core_integration.hpp"
#include "openvino/runtime/core.hpp"
#include "api_conformance_helpers.hpp"
using namespace ov::test::behavior;
using namespace ov::test::conformance;
using namespace InferenceEngine::PluginConfigParams;
namespace {
//
// IE Class Common tests with <pluginName, deviceName params>
//
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassCommon, OVClassBasicTestP,
::testing::Values(std::make_pair(getPluginLibNameByDevice(ConformanceTests::targetDevice), ConformanceTests::targetDevice)));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassNetworkTestP, OVClassNetworkTestP,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassImportExportTestP, OVClassImportExportTestP,
::testing::Values(generateComplexDeviceName(ConformanceTests::targetDevice)));
//
// IE Class GetMetric
//
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetMetricTest, OVClassGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetMetricTest, OVClassGetMetricTest_SUPPORTED_METRICS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetMetricTest, OVClassGetMetricTest_AVAILABLE_DEVICES,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetMetricTest, OVClassGetMetricTest_FULL_DEVICE_NAME,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetMetricTest, OVClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetMetricTest, OVClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetMetricTest, OVClassGetMetricTest_RANGE_FOR_STREAMS,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetMetricTest, OVClassGetMetricTest_ThrowUnsupported,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetConfigTest, OVClassGetConfigTest_ThrowUnsupported,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetAvailableDevices, OVClassGetAvailableDevices,
::testing::Values(ConformanceTests::targetDevice));
//
// IE Class GetConfig
//
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassGetConfigTest, OVClassGetConfigTest,
::testing::Values(ConformanceTests::targetDevice));
// IE Class Query network
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassQueryNetworkTest, OVClassQueryNetworkTest,
::testing::Values(ConformanceTests::targetDevice));
// IE Class Load network
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassLoadNetworkTest, OVClassLoadNetworkTest,
::testing::Values(ConformanceTests::targetDevice));
} // namespace

132
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/configuration_tests.cpp Normal file
View File

@ -0,0 +1,132 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "ie_plugin_config.hpp"
#include "ie_system_conf.h"
#include "behavior/plugin/configuration_tests.hpp"
#include "api_conformance_helpers.hpp"
using namespace BehaviorTestsDefinitions;
using namespace ov::test::conformance;
namespace {
#if (defined(__APPLE__) || defined(_WIN32))
auto defaultBindThreadParameter = InferenceEngine::Parameter{[] {
auto numaNodes = InferenceEngine::getAvailableNUMANodes();
if (numaNodes.size() > 1) {
return std::string{CONFIG_VALUE(NUMA)};
} else {
return std::string{CONFIG_VALUE(NO)};
}
}()};
#else
auto defaultBindThreadParameter = InferenceEngine::Parameter{std::string{CONFIG_VALUE(YES)}};
#endif
const std::vector<std::map<std::string, std::string>> pluginConfigs = {
{},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "10"}},
// check that hints doesn't override customer value (now for streams and later for other config opts)
};
const std::vector<std::map<std::string, std::string>> pluginMultiConfigs = {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, CorrectConfigTests,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(pluginConfigs)),
CorrectConfigTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, CorrectConfigTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI, pluginMultiConfigs))),
CorrectConfigTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, CorrectConfigTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO, pluginMultiConfigs))),
CorrectConfigTests::getTestCaseName);
const std::vector<std::map<std::string, std::string>> inPluginConfigs = {
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, "DOESN'T EXIST"}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}}
};
const std::vector<std::map<std::string, std::string>> pluginMultiInConfigs = {
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, "DOESN'T EXIST"}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, IncorrectConfigTests,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(inPluginConfigs)),
IncorrectConfigTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, IncorrectConfigTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI, pluginMultiInConfigs))),
IncorrectConfigTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, IncorrectConfigTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO, pluginMultiInConfigs))),
IncorrectConfigTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, IncorrectConfigAPITests,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(inPluginConfigs)),
IncorrectConfigAPITests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, IncorrectConfigAPITests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI, pluginMultiInConfigs))),
IncorrectConfigAPITests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, IncorrectConfigAPITests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO, pluginMultiInConfigs))),
IncorrectConfigAPITests::getTestCaseName);
const std::vector<std::map<std::string, std::string>> pluginConfigsCheck = {
{},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "10"}}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, CorrectConfigCheck,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(pluginConfigsCheck)),
CorrectConfigCheck::getTestCaseName);
} // namespace

103
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/core_integration.cpp Normal file
View File

@ -0,0 +1,103 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/plugin/core_integration.hpp"
#include "api_conformance_helpers.hpp"
using namespace BehaviorTestsDefinitions;
using namespace ov::test::conformance;
using namespace InferenceEngine::PluginConfigParams;
namespace {
//
// IE Class Common tests with <pluginName, deviceName params>
//
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassCommon, IEClassBasicTestP,
::testing::Values(std::make_pair(getPluginLibNameByDevice(ConformanceTests::targetDevice), ConformanceTests::targetDevice)));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassNetworkTestP, IEClassNetworkTestP,
::testing::Values(ConformanceTests::targetDevice));
//
// IE Class GetMetric
//
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_METRICS,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_AVAILABLE_DEVICES,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_FULL_DEVICE_NAME,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_RANGE_FOR_STREAMS,
::testing::Values(ConformanceTests::targetDevice));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_ThrowUnsupported,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetConfigTest, IEClassGetConfigTest_ThrowUnsupported,
::testing::ValuesIn(returnAllPossibleDeviceCombination()));
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetAvailableDevices, IEClassGetAvailableDevices,
::testing::Values(ConformanceTests::targetDevice));
//
// IE Class GetConfig
//
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetConfigTest, IEClassGetConfigTest,
::testing::Values(ConformanceTests::targetDevice));
//////////////////////////////////////////////////////////////////////////////////////////
TEST(IEClassBasicTest, smoke_SetConfigAfterCreatedThrow) {
InferenceEngine::Core ie;
std::string value = {};
ASSERT_NO_THROW(ie.SetConfig({{KEY_CPU_THREADS_NUM, "1"}}, ConformanceTests::targetDevice));
ASSERT_NO_THROW(value = ie.GetConfig(ConformanceTests::targetDevice, KEY_CPU_THREADS_NUM).as<std::string>());
ASSERT_EQ("1", value);
ASSERT_NO_THROW(ie.SetConfig({{KEY_CPU_THREADS_NUM, "4"}}, ConformanceTests::targetDevice));
ASSERT_NO_THROW(value = ie.GetConfig(ConformanceTests::targetDevice, KEY_CPU_THREADS_NUM).as<std::string>());
ASSERT_EQ("4", value);
}
// IE Class Query network
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassQueryNetworkTest, IEClassQueryNetworkTest,
::testing::Values(ConformanceTests::targetDevice));
// IE Class Load network
INSTANTIATE_TEST_SUITE_P(
smoke_IEClassLoadNetworkTest, IEClassLoadNetworkTest,
::testing::Values(ConformanceTests::targetDevice));
} // namespace

31
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/core_threading_tests.cpp Normal file
View File

@ -0,0 +1,31 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <behavior/plugin/core_threading.hpp>
#include "api_conformance_helpers.hpp"
namespace {
using namespace ov::test::conformance;
const Params coreThreadingParams[] = {
std::tuple<Device, Config>{ CommonTestUtils::DEVICE_HETERO, generateConfigs(CommonTestUtils::DEVICE_HETERO).front() },
std::tuple<Device, Config>{ CommonTestUtils::DEVICE_MULTI, generateConfigs(CommonTestUtils::DEVICE_MULTI).front() },
std::tuple<Device, Config>{ CommonTestUtils::DEVICE_AUTO, generateConfigs(CommonTestUtils::DEVICE_AUTO).front() },
};
INSTANTIATE_TEST_SUITE_P(Conformance_, CoreThreadingTests, testing::ValuesIn(coreThreadingParams), CoreThreadingTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(Conformance, CoreThreadingTests,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::Values(Config{{ CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) }})),
CoreThreadingTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(Conformance, CoreThreadingTestsWithIterations,
testing::Combine(testing::ValuesIn(coreThreadingParams),
testing::Values(4),
testing::Values(50),
testing::Values(ModelClass::Default)),
CoreThreadingTestsWithIterations::getTestCaseName);
} // namespace

32
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/life_time.cpp Normal file
View File

@ -0,0 +1,32 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/plugin/life_time.hpp"
#include "api_conformance_helpers.hpp"
using namespace BehaviorTestsDefinitions;
using namespace ov::test::conformance;
namespace {
const std::vector<std::vector<int >> orders = {
// 0 - plugin
// 1 - executable_network
// 2 - infer_request
// 3 - variable state
{3, 0, 1, 2},
{3, 0, 2, 1},
{3, 1, 0, 2},
{3, 1, 2, 0},
{3, 2, 0, 1},
{3, 2, 1, 0},
{0, 3, 1, 2},
{0, 1, 3, 2}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, HoldersTest,
::testing::Combine(
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(orders)),
HoldersTest::getTestCaseName);
} // namespace

27
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/version.cpp Normal file
View File

@ -0,0 +1,27 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "behavior/plugin/version.hpp"
#include "api_conformance_helpers.hpp"
using namespace BehaviorTestsDefinitions;
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, VersionTest,
::testing::Values(ConformanceTests::targetDevice),
VersionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, VersionTest,
::testing::Values(CommonTestUtils::DEVICE_MULTI),
VersionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, VersionTest,
::testing::Values(CommonTestUtils::DEVICE_AUTO),
VersionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, VersionTest,
::testing::Values(CommonTestUtils::DEVICE_HETERO),
VersionTest::getTestCaseName);
} // namespace

149
inference-engine/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/preprocessing/set_preprocess.cpp
View File

@ -3,14 +3,14 @@
//
#include "behavior/preprocessing/set_preprocess.hpp"
#include "conformance.hpp"
#include "api_conformance_helpers.hpp"
namespace {
using namespace ConformanceTests;
using namespace BehaviorTestsDefinitions;
using namespace ov::test::conformance;
const std::vector<InferenceEngine::Precision> netPrecisions = {
const std::vector<InferenceEngine::Precision> netPrecisionsPreprocess = {
InferenceEngine::Precision::FP32,
InferenceEngine::Precision::FP16
};
@ -19,171 +19,158 @@ const std::vector<std::map<std::string, std::string>> configs = {
{},
};
const std::vector<std::map<std::string, std::string>> heteroConfigs = {
{{ "TARGET_FALLBACK" , targetDevice}}
};
const std::vector<std::map<std::string, std::string>> multiConfigs = {
{{ InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , targetDevice}}
};
const std::vector<std::map<std::string, std::string>> autoConfigs = {
{{ InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , targetDevice}}
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(targetDevice),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(configs)),
InferRequestPreprocessTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestPreprocessTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(heteroConfigs)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestPreprocessTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestPreprocessTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(multiConfigs)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestPreprocessTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestPreprocessTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(autoConfigs)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestPreprocessTest::getTestCaseName);
const std::vector<InferenceEngine::Precision> ioPrecisions = {
const std::vector<InferenceEngine::Precision> ioPrecisionsPreprocess = {
InferenceEngine::Precision::FP32,
InferenceEngine::Precision::U8
};
const std::vector<InferenceEngine::Layout> netLayouts = {
const std::vector<InferenceEngine::Layout> netLayoutsPreprocess = {
InferenceEngine::Layout::NCHW,
// InferenceEngine::Layout::NHWC
};
const std::vector<InferenceEngine::Layout> ioLayouts = {
const std::vector<InferenceEngine::Layout> ioLayoutsPreprocess = {
InferenceEngine::Layout::NCHW,
InferenceEngine::Layout::NHWC
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessConversionTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(ioPrecisions),
::testing::ValuesIn(ioPrecisions),
::testing::ValuesIn(netLayouts),
::testing::ValuesIn(ioLayouts),
::testing::ValuesIn(ioLayouts),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::ValuesIn(ioPrecisionsPreprocess),
::testing::ValuesIn(ioPrecisionsPreprocess),
::testing::ValuesIn(netLayoutsPreprocess),
::testing::ValuesIn(ioLayoutsPreprocess),
::testing::ValuesIn(ioLayoutsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::Values(targetDevice),
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(configs)),
InferRequestPreprocessConversionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessDynamicallyInSetBlobTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::ValuesIn(netLayouts),
::testing::ValuesIn(netLayoutsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::Values(true), // only SetBlob
::testing::Values(true), // only SetBlob
::testing::Values(targetDevice),
::testing::Values(ConformanceTests::targetDevice),
::testing::ValuesIn(configs)),
InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestPreprocessConversionTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(ioPrecisions),
::testing::ValuesIn(ioPrecisions),
::testing::ValuesIn(netLayouts),
::testing::ValuesIn(ioLayouts),
::testing::ValuesIn(ioLayouts),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::ValuesIn(ioPrecisionsPreprocess),
::testing::ValuesIn(ioPrecisionsPreprocess),
::testing::ValuesIn(netLayoutsPreprocess),
::testing::ValuesIn(ioLayoutsPreprocess),
::testing::ValuesIn(ioLayoutsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(heteroConfigs)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestPreprocessConversionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestPreprocessDynamicallyInSetBlobTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::ValuesIn(netLayouts),
::testing::ValuesIn(netLayoutsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::Values(true), // only SetBlob
::testing::Values(true), // only SetBlob
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(heteroConfigs)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_HETERO))),
InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestPreprocessConversionTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(ioPrecisions),
::testing::ValuesIn(ioPrecisions),
::testing::ValuesIn(netLayouts),
::testing::ValuesIn(ioLayouts),
::testing::ValuesIn(ioLayouts),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::ValuesIn(ioPrecisionsPreprocess),
::testing::ValuesIn(ioPrecisionsPreprocess),
::testing::ValuesIn(netLayoutsPreprocess),
::testing::ValuesIn(ioLayoutsPreprocess),
::testing::ValuesIn(ioLayoutsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(multiConfigs)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestPreprocessConversionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestPreprocessDynamicallyInSetBlobTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::ValuesIn(netLayouts),
::testing::ValuesIn(netLayoutsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::Values(true), // only SetBlob
::testing::Values(true), // only SetBlob
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(multiConfigs)),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_MULTI))),
InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestPreprocessConversionTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::ValuesIn(ioPrecisions),
::testing::ValuesIn(ioPrecisions),
::testing::ValuesIn(netLayouts),
::testing::ValuesIn(ioLayouts),
::testing::ValuesIn(ioLayouts),
::testing::Bool(),
::testing::Bool(),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(autoConfigs)),
InferRequestPreprocessConversionTest::getTestCaseName);
::testing::Combine(
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::ValuesIn(ioPrecisionsPreprocess),
::testing::ValuesIn(ioPrecisionsPreprocess),
::testing::ValuesIn(netLayoutsPreprocess),
::testing::ValuesIn(ioLayoutsPreprocess),
::testing::ValuesIn(ioLayoutsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestPreprocessConversionTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestPreprocessDynamicallyInSetBlobTest,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Bool(),
::testing::Bool(),
::testing::ValuesIn(netLayouts),
::testing::Bool(),
::testing::Bool(),
::testing::Values(true), // only SetBlob
::testing::Values(true), // only SetBlob
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(autoConfigs)),
InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName);
::testing::Combine(
::testing::ValuesIn(netPrecisionsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::ValuesIn(netLayoutsPreprocess),
::testing::Bool(),
::testing::Bool(),
::testing::Values(true), // only SetBlob
::testing::Values(true), // only SetBlob
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generateConfigs(CommonTestUtils::DEVICE_AUTO))),
InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName);
} // namespace