IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[AUTO] Clean up the logic of AUTO device priorities (#18557)

Browse Source 
* Update selection logic for 3rd part devices within AUTO plugin.

* Remove MYRAID from AUTO test case.

* Update.

* Update.

* Update.

---------

Signed-off-by: Wang, Yang <yang4.wang@intel.com>
Co-authored-by: Chen Peter <peter.chen@intel.com>
This commit is contained in:
Wang, Yang 2023-07-25 02:03:14 +00:00 committed by GitHub
parent dfb1493f2f
commit 85f514788e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 319 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
src/plugins/auto/src/plugin.cpp
View File

@ -602,91 +602,73 @@ std::list<DeviceInformation> Plugin::get_valid_device(
if (meta_devices.empty()) {
OPENVINO_THROW("No available device to select in ", get_device_name());
}
bool is_default_list = true;
std::list<DeviceInformation> valid_devices;
std::list<DeviceInformation> CPU;
std::list<DeviceInformation> dGPU;
std::list<DeviceInformation> iGPU;
std::list<DeviceInformation> MYRIAD;
std::list<DeviceInformation> VPU;
std::list<DeviceInformation> Others;
auto is_supported_model = [this, model_precision](const std::string& device_name) {
// Check if candidate device supported the specified model precision.
auto capability = get_core()->get_property(device_name, ov::device::capabilities);
auto support_model = std::find(capability.begin(), capability.end(), (model_precision));
bool is_support_fp16 =
model_precision == "FP32" && std::find(capability.begin(), capability.end(), ("FP16")) != capability.end();
return support_model != capability.end() || is_support_fp16;
};
for (auto& item : meta_devices) {
if (item.device_name.find("CPU") == 0) {
CPU.push_back(item);
for (auto&& device_info : meta_devices) {
if (device_info.device_priority > 0)
is_default_list = false;
// check if device support this model precision
if (!is_supported_model(device_info.device_name) && meta_devices.size() > 1)
continue;
}
if (item.device_name.find("MYRIAD") == 0) {
MYRIAD.push_back(item);
continue;
}
if (item.device_name.find("VPU") == 0) {
VPU.push_back(item);
continue;
}
if (item.device_name.find("GPU") == 0) {
if (device_info.device_name.find("CPU") == 0) {
CPU.push_back(device_info);
} else if (device_info.device_name.find("GPU") == 0) {
std::string device_type;
try {
// can optimize to typed function when gpu swith to 2.0 api
device_type = get_core()->get_property(item.device_name, ov::device::type.name(), {}).as<std::string>();
device_type =
get_core()->get_property(device_info.device_name, ov::device::type.name(), {}).as<std::string>();
} catch (const ov::Exception&) {
LOG_DEBUG_TAG("get property :%s for %s failed ", "DEVICE_TYPE", item.device_name.c_str());
LOG_DEBUG_TAG("get property :%s for %s failed ", "DEVICE_TYPE", device_info.device_name.c_str());
}
if (device_type == "integrated") {
iGPU.push_back(item);
iGPU.push_back(device_info);
} else if (device_type == "discrete") {
dGPU.push_back(item);
dGPU.push_back(device_info);
} else {
LOG_DEBUG_TAG("Unknown device type for %s", item.device_name.c_str());
}
continue;
}
}
// Priority of selecting device: dGPU > VPU > iGPU > MYRIAD > CPU
std::list<DeviceInformation> devices;
if (model_precision == "INT8") {
devices.splice(devices.end(), VPU);
devices.splice(devices.end(), dGPU);
} else {
devices.splice(devices.end(), dGPU);
devices.splice(devices.end(), VPU);
}
devices.splice(devices.end(), iGPU);
devices.splice(devices.end(), MYRIAD);
devices.splice(devices.end(), CPU);
std::list<DeviceInformation> valid_devices;
if (meta_devices.size() > 1) {
auto select_support_dev = [this, &devices, &valid_devices](const std::string& model_precision) {
for (auto iter = devices.begin(); iter != devices.end();) {
auto capability = get_core()->get_property(iter->device_name, ov::device::capabilities);
auto support_model = std::find(capability.begin(), capability.end(), (model_precision));
if (support_model != capability.end()) {
valid_devices.push_back(std::move(*iter));
devices.erase(iter++);
continue;
}
iter++;
}
};
select_support_dev(model_precision);
// If model is FP32, continue to collect the device support FP16 but not support FP32.
if (model_precision == "FP32") {
const std::string f16 = "FP16";
select_support_dev(f16);
LOG_DEBUG_TAG("Unknown device type for %s", device_info.device_name.c_str());
}
} else {
valid_devices.push_back(meta_devices[0]);
Others.push_back(device_info);
}
valid_devices.push_back(device_info);
}
if (valid_devices.empty()) {
OPENVINO_THROW("Cannot select any device");
}
if (valid_devices.size() == 1) {
return valid_devices;
}
if (is_default_list) {
// Generate the default device priority for selecting logic of AUTO.
// Default priority of selecting device: dGPU > iGPU > 3rd part devices > CPU
valid_devices.clear();
valid_devices.splice(valid_devices.end(), dGPU);
valid_devices.splice(valid_devices.end(), iGPU);
valid_devices.splice(valid_devices.end(), Others);
valid_devices.splice(valid_devices.end(), CPU);
return valid_devices;
}
// sort validDevices
valid_devices.sort([](const DeviceInformation& a, const DeviceInformation& b) {
return a.device_priority < b.device_priority;
});
return valid_devices;
}

11
src/plugins/auto/tests/unit/auto_unit_test.cpp
View File

@ -69,8 +69,7 @@ ov::mock_auto_plugin::tests::AutoTest::AutoTest() {
ON_CALL(*core, get_property(_, StrEq(ov::compilation_num_threads.name()), _)).WillByDefault(Return(12));
std::vector<std::string> cpuCability = {"FP32", "FP16", "INT8", "BIN"};
std::vector<std::string> gpuCability = {"FP32", "FP16", "BATCHED_BLOB", "BIN", "INT8"};
std::vector<std::string> vpuCability = {"INT8"};
std::vector<std::string> myriadCability = {"FP16"};
std::vector<std::string> othersCability = {"FP32", "FP16"};
std::string igpuArchitecture = "GPU: vendor=0x8086 arch=0";
std::string dgpuArchitecture = "GPU: vendor=0x8086 arch=1";
auto iGpuType = ov::device::Type::INTEGRATED;
@ -79,10 +78,8 @@ ov::mock_auto_plugin::tests::AutoTest::AutoTest() {
StrEq(ov::device::capabilities.name()), _)).WillByDefault(RETURN_MOCK_VALUE(cpuCability));
ON_CALL(*core, get_property(HasSubstr("GPU"),
StrEq(ov::device::capabilities.name()), _)).WillByDefault(RETURN_MOCK_VALUE(gpuCability));
ON_CALL(*core, get_property(StrEq(CommonTestUtils::DEVICE_KEEMBAY),
StrEq(ov::device::capabilities.name()), _)).WillByDefault(RETURN_MOCK_VALUE(vpuCability));
ON_CALL(*core, get_property(StrEq("MYRIAD"),
StrEq(ov::device::capabilities.name()), _)).WillByDefault(RETURN_MOCK_VALUE(myriadCability));
ON_CALL(*core, get_property(StrEq("OTHERS"),
StrEq(ov::device::capabilities.name()), _)).WillByDefault(RETURN_MOCK_VALUE(othersCability));
ON_CALL(*core, get_property(StrEq("GPU"),
StrEq(ov::device::architecture.name()), _)).WillByDefault(RETURN_MOCK_VALUE(igpuArchitecture));
ON_CALL(*core, get_property(StrEq("GPU.0"),
@ -108,7 +105,7 @@ ov::mock_auto_plugin::tests::AutoTest::AutoTest() {
StrEq(ov::device::full_name.name()), _)).WillByDefault(RETURN_MOCK_VALUE(igpuFullDeviceName));
ON_CALL(*core, get_property(StrEq("GPU.1"),
StrEq(ov::device::full_name.name()), _)).WillByDefault(RETURN_MOCK_VALUE(dgpuFullDeviceName));
const std::vector<std::string> availableDevs = {"CPU", "GPU.0", "GPU.1", "VPU"};
const std::vector<std::string> availableDevs = {"CPU", "GPU.0", "GPU.1"};
ON_CALL(*core, get_available_devices()).WillByDefault(Return(availableDevs));
ON_CALL(*plugin, parse_meta_devices)
.WillByDefault(

84
src/plugins/auto/tests/unit/key_network_priority_test.cpp
View File

@ -48,6 +48,10 @@ public:
std::vector<std::string> gpuCability = {"FP32", "FP16", "BATCHED_BLOB", "BIN"};
ON_CALL(*core, get_property(HasSubstr("GPU"),
StrEq(ov::device::capabilities.name()), _)).WillByDefault(RETURN_MOCK_VALUE(gpuCability));
std::vector<std::string> otherCability = {"INT8"};
ON_CALL(*core, get_property(HasSubstr("OTHER"), StrEq(ov::device::capabilities.name()), _))
.WillByDefault(RETURN_MOCK_VALUE(otherCability));
ON_CALL(*plugin, get_valid_device)
.WillByDefault([this](const std::vector<DeviceInformation>& metaDevices, const std::string& netPrecision) {
return plugin->Plugin::get_valid_device(metaDevices, netPrecision);
@ -67,14 +71,12 @@ TEST_P(KeyNetworkPriorityTest, SelectDevice) {
metaDevices = {{CommonTestUtils::DEVICE_CPU, {}, 2, "", "CPU_01", 0},
{"GPU.0", {}, 2, "01", "iGPU_01", 1},
{"GPU.1", {}, 2, "01", "dGPU_01", 2},
{"MYRIAD", {}, 2, "01", "MYRIAD_01", 3},
{CommonTestUtils::DEVICE_KEEMBAY, {}, 2, "01", "VPU_01", 4}};
{"OTHER", {}, 2, "01", "OTHER_01", 3}};
} else {
metaDevices = {{CommonTestUtils::DEVICE_CPU, {}, 2, "", "CPU_01", 0},
{"GPU.0", {}, 2, "01", "iGPU_01", 0},
{"GPU.1", {}, 2, "01", "dGPU_01", 0},
{"MYRIAD", {}, 2, "01", "MYRIAD_01", 0},
{CommonTestUtils::DEVICE_KEEMBAY, {}, 2, "01", "VPU_01", 0}};
{"OTHER", {}, 2, "01", "OTHER_01", 0}};
}
EXPECT_CALL(*plugin, select_device(_, _, _)).Times(sizeOfConfigs);
@ -96,14 +98,12 @@ TEST_P(KeyNetworkPriorityTest, MultiThreadsSelectDevice) {
metaDevices = {{CommonTestUtils::DEVICE_CPU, {}, 2, "", "CPU_01", 0},
{"GPU.0", {}, 2, "01", "iGPU_01", 1},
{"GPU.1", {}, 2, "01", "dGPU_01", 2},
{"MYRIAD", {}, 2, "01", "MYRIAD_01", 3},
{CommonTestUtils::DEVICE_KEEMBAY, {}, 2, "01", "VPU_01", 4}};
{"OTHER", {}, 2, "01", "OTHER_01", 3}};
} else {
metaDevices = {{CommonTestUtils::DEVICE_CPU, {}, 2, "", "CPU_01", 0},
{"GPU.0", {}, 2, "01", "iGPU_01", 0},
{"GPU.1", {}, 2, "01", "dGPU_01", 0},
{"MYRIAD", {}, 2, "01", "MYRIAD_01", 0},
{CommonTestUtils::DEVICE_KEEMBAY, {}, 2, "01", "VPU_01", 0}};
{"OTHER", {}, 2, "01", "OTHER_01", 0}};
}
EXPECT_CALL(*plugin, select_device(_, _, _)).Times(sizeOfConfigs * 2);
@ -136,8 +136,6 @@ TEST_P(KeyNetworkPriorityTest, MultiThreadsSelectDevice) {
// example
// ConfigParams {"FP32", false, {PriorityParams {0, "dGPU_01"},
// PriorityParams {1, "iGPU_01"},
// PriorityParams {2, "MYRIAD_01"},
// PriorityParams {2, "MYRIAD_01"}}},
// {netPrecision, enableDevicePriority, PriorityParamsVector{{modelpriority, expect device unique_name}}}
const std::vector<ConfigParams> testConfigs = {
@ -147,8 +145,7 @@ const std::vector<ConfigParams> testConfigs = {
PriorityParams {2, "CPU_01"}}},
ConfigParams {"FP32", false, {PriorityParams {2, "dGPU_01"},
PriorityParams {3, "iGPU_01"},
PriorityParams {4, "CPU_01"},
PriorityParams {5, "MYRIAD_01"}}},
PriorityParams {4, "CPU_01"}}},
ConfigParams {"FP32", false, {PriorityParams {2, "dGPU_01"},
PriorityParams {0, "dGPU_01"},
PriorityParams {2, "iGPU_01"},
@ -160,32 +157,30 @@ const std::vector<ConfigParams> testConfigs = {
ConfigParams {"FP32", false, {PriorityParams {0, "dGPU_01"},
PriorityParams {1, "iGPU_01"},
PriorityParams {2, "CPU_01"},
PriorityParams {3, "MYRIAD_01"},
PriorityParams {0, "dGPU_01"},
PriorityParams {1, "iGPU_01"},
PriorityParams {2, "CPU_01"},
PriorityParams {3, "MYRIAD_01"}}},
ConfigParams {"INT8", false, {PriorityParams {0, "VPU_01"},
PriorityParams {2, "CPU_01"}}},
ConfigParams {"INT8", false, {PriorityParams {0, "OTHER_01"},
PriorityParams {1, "CPU_01"},
PriorityParams {2, "CPU_01"},
PriorityParams {2, "CPU_01"}}},
ConfigParams {"INT8", false, {PriorityParams {2, "VPU_01"},
ConfigParams {"INT8", false, {PriorityParams {2, "OTHER_01"},
PriorityParams {3, "CPU_01"},
PriorityParams {4, "CPU_01"},
PriorityParams {5, "CPU_01"}}},
ConfigParams {"INT8", false, {PriorityParams {2, "VPU_01"},
PriorityParams {0, "VPU_01"},
ConfigParams {"INT8", false, {PriorityParams {2, "OTHER_01"},
PriorityParams {0, "OTHER_01"},
PriorityParams {2, "CPU_01"},
PriorityParams {2, "CPU_01"}}},
ConfigParams {"INT8", false, {PriorityParams {2, "VPU_01"},
PriorityParams {0, "VPU_01"},
ConfigParams {"INT8", false, {PriorityParams {2, "OTHER_01"},
PriorityParams {0, "OTHER_01"},
PriorityParams {2, "CPU_01"},
PriorityParams {3, "CPU_01"}}},
ConfigParams {"INT8", false, {PriorityParams {0, "VPU_01"},
ConfigParams {"INT8", false, {PriorityParams {0, "OTHER_01"},
PriorityParams {1, "CPU_01"},
PriorityParams {2, "CPU_01"},
PriorityParams {3, "CPU_01"},
PriorityParams {0, "VPU_01"},
PriorityParams {0, "OTHER_01"},
PriorityParams {1, "CPU_01"},
PriorityParams {2, "CPU_01"},
PriorityParams {3, "CPU_01"}}},
@ -216,17 +211,14 @@ const std::vector<ConfigParams> testConfigs = {
// metaDevices = {{CommonTestUtils::DEVICE_CPU, {}, 2, "", "CPU_01", 0},
// {CommonTestUtils::DEVICE_GPU, {}, 2, "01", "iGPU_01", 1},
// {CommonTestUtils::DEVICE_GPU, {}, 2, "01", "dGPU_01", 2},
// {"MYRIAD", {}, 2, "01", "MYRIAD_01", 3},
// {CommonTestUtils::DEVICE_KEEMBAY, {}, 2, "01", "VPU_01", 4}};
// cpu > igpu > dgpu > MYRIAD > VPU
// cpu > igpu > dgpu > OTHER
ConfigParams {"FP32", true, {PriorityParams {0, "CPU_01"},
PriorityParams {1, "iGPU_01"},
PriorityParams {2, "dGPU_01"},
PriorityParams {2, "dGPU_01"}}},
ConfigParams {"FP32", true, {PriorityParams {2, "CPU_01"},
PriorityParams {3, "iGPU_01"},
PriorityParams {4, "dGPU_01"},
PriorityParams {5, "MYRIAD_01"}}},
PriorityParams {4, "dGPU_01"}}},
ConfigParams {"FP32", true, {PriorityParams {2, "CPU_01"},
PriorityParams {0, "CPU_01"},
PriorityParams {2, "iGPU_01"},
@ -238,35 +230,33 @@ const std::vector<ConfigParams> testConfigs = {
ConfigParams {"FP32", true, {PriorityParams {0, "CPU_01"},
PriorityParams {1, "iGPU_01"},
PriorityParams {2, "dGPU_01"},
PriorityParams {3, "MYRIAD_01"},
PriorityParams {0, "CPU_01"},
PriorityParams {1, "iGPU_01"},
PriorityParams {2, "dGPU_01"},
PriorityParams {3, "MYRIAD_01"}}},
PriorityParams {2, "dGPU_01"}}},
ConfigParams {"INT8", true, {PriorityParams {0, "CPU_01"},
PriorityParams {1, "VPU_01"},
PriorityParams {2, "VPU_01"},
PriorityParams {2, "VPU_01"}}},
PriorityParams {1, "OTHER_01"},
PriorityParams {2, "OTHER_01"},
PriorityParams {2, "OTHER_01"}}},
ConfigParams {"INT8", true, {PriorityParams {2, "CPU_01"},
PriorityParams {3, "VPU_01"},
PriorityParams {4, "VPU_01"},
PriorityParams {5, "VPU_01"}}},
PriorityParams {3, "OTHER_01"},
PriorityParams {4, "OTHER_01"},
PriorityParams {5, "OTHER_01"}}},
ConfigParams {"INT8", true, {PriorityParams {2, "CPU_01"},
PriorityParams {0, "CPU_01"},
PriorityParams {2, "VPU_01"},
PriorityParams {2, "VPU_01"}}},
PriorityParams {2, "OTHER_01"},
PriorityParams {2, "OTHER_01"}}},
ConfigParams {"INT8", true, {PriorityParams {2, "CPU_01"},
PriorityParams {0, "CPU_01"},
PriorityParams {2, "VPU_01"},
PriorityParams {3, "VPU_01"}}},
PriorityParams {2, "OTHER_01"},
PriorityParams {3, "OTHER_01"}}},
ConfigParams {"INT8", true, {PriorityParams {0, "CPU_01"},
PriorityParams {1, "VPU_01"},
PriorityParams {2, "VPU_01"},
PriorityParams {3, "VPU_01"},
PriorityParams {1, "OTHER_01"},
PriorityParams {2, "OTHER_01"},
PriorityParams {3, "OTHER_01"},
PriorityParams {0, "CPU_01"},
PriorityParams {1, "VPU_01"},
PriorityParams {2, "VPU_01"},
PriorityParams {3, "VPU_01"}}},
PriorityParams {1, "OTHER_01"},
PriorityParams {2, "OTHER_01"},
PriorityParams {3, "OTHER_01"}}},
ConfigParams {"BIN", true, {PriorityParams {0, "CPU_01"},
PriorityParams {1, "iGPU_01"},
PriorityParams {2, "dGPU_01"},

34
src/plugins/auto/tests/unit/parse_meta_device_test.cpp
View File

@ -10,7 +10,7 @@ using testing::Throw;
const char igpuFullDeviceName[] = "Intel(R) Gen9 HD Graphics (iGPU)";
const char dgpuFullDeviceName[] = "Intel(R) Iris(R) Xe MAX Graphics (dGPU)";
const std::vector<std::string> availableDevsNoID = {"CPU", "GPU", "VPU"};
const std::vector<std::string> availableDevsNoID = {"CPU", "GPU", "OTHER"};
using ConfigParams = std::tuple<std::string, // Priority devices
std::vector<DeviceInformation>, // expect metaDevices
bool, // if throw exception
@ -127,57 +127,57 @@ TEST_P(ParseMetaDeviceNoIDTest, ParseMetaDevices) {
// ConfigParams {devicePriority, expect metaDevices, ifThrowException}
const std::vector<ConfigParams> testConfigs = {
ConfigParams{"CPU,GPU,VPU",
ConfigParams{"CPU,GPU,OTHER",
{{"CPU", {}, -1, "", "CPU_", 0},
{"GPU.0", {}, -1, "", std::string(igpuFullDeviceName) + "_0", 1},
{"GPU.1", {}, -1, "", std::string(dgpuFullDeviceName) + "_1", 1},
{"VPU", {}, -1, "", "VPU_", 2}},
{"OTHER", {}, -1, "", "OTHER_", 2}},
false,
4},
ConfigParams{"VPU,GPU,CPU",
{{"VPU", {}, -1, "", "VPU_", 0},
ConfigParams{"OTHER,GPU,CPU",
{{"OTHER", {}, -1, "", "OTHER_", 0},
{"GPU.0", {}, -1, "", std::string(igpuFullDeviceName) + "_0", 1},
{"GPU.1", {}, -1, "", std::string(dgpuFullDeviceName) + "_1", 1},
{"CPU", {}, -1, "", "CPU_", 2}},
false,
4},
ConfigParams{"VPU,GPU,INVALID_DEVICE",
{{"VPU", {}, -1, "", "VPU_", 0},
ConfigParams{"OTHER,GPU,INVALID_DEVICE",
{{"OTHER", {}, -1, "", "OTHER_", 0},
{"GPU.0", {}, -1, "", std::string(igpuFullDeviceName) + "_0", 1},
{"GPU.1", {}, -1, "", std::string(dgpuFullDeviceName) + "_1", 1}},
false,
3},
ConfigParams{"CPU(1),GPU(2),VPU(4)",
ConfigParams{"CPU(1),GPU(2),OTHER(4)",
{{"CPU", {}, 1, "", "CPU_", 0},
{"GPU.0", {}, 2, "", std::string(igpuFullDeviceName) + "_0", 1},
{"GPU.1", {}, 2, "", std::string(dgpuFullDeviceName) + "_1", 1},
{"VPU", {}, 4, "", "VPU_", 2}},
{"OTHER", {}, 4, "", "OTHER_", 2}},
false,
4},
ConfigParams{"CPU(-1),GPU,VPU", {}, true, 0},
ConfigParams{"CPU(NA),GPU,VPU", {}, true, 0},
ConfigParams{"CPU(-1),GPU,OTHER", {}, true, 0},
ConfigParams{"CPU(NA),GPU,OTHER", {}, true, 0},
ConfigParams{"INVALID_DEVICE", {}, false, 0},
ConfigParams{"INVALID_DEVICE,CPU", {{"CPU", {}, -1, "", "CPU_", 1}}, false, 2},
ConfigParams{"CPU(3),GPU.1,VPU",
ConfigParams{"CPU(3),GPU.1,OTHER",
{{"CPU", {}, 3, "", "CPU_", 0},
{"GPU.1", {}, -1, "", std::string(dgpuFullDeviceName) + "_1", 1},
{"VPU", {}, -1, "", "VPU_", 2}},
{"OTHER", {}, -1, "", "OTHER_", 2}},
false,
3},
ConfigParams{"VPU,GPU.1,CPU(3)",
{{"VPU", {}, -1, "", "VPU_", 0},
ConfigParams{"OTHER,GPU.1,CPU(3)",
{{"OTHER", {}, -1, "", "OTHER_", 0},
{"GPU.1", {}, -1, "", std::string(dgpuFullDeviceName) + "_1", 1},
{"CPU", {}, 3, "", "CPU_", 2}},
false,
3}};
const std::vector<ConfigParams> testConfigsNoID = {
ConfigParams{"CPU,GPU,VPU",
ConfigParams{"CPU,GPU,OTHER",
{{"CPU", {}, -1, "", "CPU_", 0},
{"GPU", {}, -1, "0", std::string(igpuFullDeviceName) + "_0", 1},
{"VPU", {}, -1, "", "VPU_", 2}},
{"OTHER", {}, -1, "", "OTHER_", 2}},
false,
3},
};

63
src/plugins/auto/tests/unit/runtime_fallback_test.cpp
View File

@ -13,23 +13,23 @@ class AutoRuntimeFallback : public tests::AutoTest,
public ::testing::TestWithParam<ConfigParams> {
public:
ov::SoPtr<ov::MockCompiledModel> mockExeNetworkGPU_1;
ov::SoPtr<ov::MockCompiledModel> mockExeNetworkVPU;
ov::SoPtr<ov::MockCompiledModel> mockExeNetworkOTHER;
std::shared_ptr<NiceMock<ov::MockSyncInferRequest>> inferReqInternalGPU_1;
std::shared_ptr<NiceMock<ov::MockSyncInferRequest>> inferReqInternalVPU;
std::shared_ptr<NiceMock<ov::MockSyncInferRequest>> inferReqInternalOTHER;
std::shared_ptr<NiceMock<ov::MockCompiledModel>> mockIExeNetGPU_1;
std::shared_ptr<NiceMock<ov::MockCompiledModel>> mockIExeNetVPU;
std::shared_ptr<NiceMock<ov::MockCompiledModel>> mockIExeNetOTHER;
std::shared_ptr<ov::MockAsyncInferRequest> mockInferrequest;
std::shared_ptr<ov::MockAsyncInferRequest> mockInferrequestGPU_0;
std::shared_ptr<ov::MockAsyncInferRequest> mockInferrequestGPU_1;
std::shared_ptr<ov::MockAsyncInferRequest> mockInferrequestVPU;
std::shared_ptr<ov::MockAsyncInferRequest> mockInferrequestOTHER;
std::shared_ptr<ov::threading::ImmediateExecutor> mockExecutor;
std::shared_ptr<ov::threading::ImmediateExecutor> mockExecutorGPU_0;
std::shared_ptr<ov::threading::ImmediateExecutor> mockExecutorGPU_1;
std::shared_ptr<ov::threading::ImmediateExecutor> mockExecutorVPU;
std::shared_ptr<ov::threading::ImmediateExecutor> mockExecutorOTHER;
public:
static std::string getTestCaseName(testing::TestParamInfo<ConfigParams> obj) {
@ -66,15 +66,15 @@ public:
void TearDown() override {
mockExeNetworkGPU_1 = {};
inferReqInternalGPU_1.reset();
inferReqInternalVPU.reset();
inferReqInternalOTHER.reset();
mockIExeNetGPU_1.reset();
mockIExeNetVPU.reset();
mockIExeNetOTHER.reset();
mockIExeNetGPU_1.reset();
mockIExeNetVPU.reset();
mockIExeNetOTHER.reset();
mockExecutor.reset();
mockExecutorGPU_0.reset();
mockExecutorGPU_1.reset();
mockExecutorVPU.reset();
mockExecutorOTHER.reset();
}
void SetUp() override {
@ -82,8 +82,8 @@ public:
mockIExeNetGPU_1 = std::make_shared<NiceMock<ov::MockCompiledModel>>(model, plugin);
mockExeNetworkGPU_1 = {mockIExeNetGPU_1, {}};
mockIExeNetVPU = std::make_shared<NiceMock<ov::MockCompiledModel>>(model, plugin);
mockExeNetworkVPU = {mockIExeNetVPU, {}};
mockIExeNetOTHER = std::make_shared<NiceMock<ov::MockCompiledModel>>(model, plugin);
mockExeNetworkOTHER = {mockIExeNetOTHER, {}};
// prepare mockicore and cnnNetwork for loading
ON_CALL(*core, compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
@ -95,9 +95,9 @@ public:
std::this_thread::sleep_for(std::chrono::milliseconds(200));
return mockExeNetworkGPU_1; }));
ON_CALL(*core, compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
::testing::Matcher<const std::string&>(StrEq(CommonTestUtils::DEVICE_KEEMBAY)), _)).WillByDefault(InvokeWithoutArgs([this]() {
::testing::Matcher<const std::string&>(StrEq("OTHER")), _)).WillByDefault(InvokeWithoutArgs([this]() {
std::this_thread::sleep_for(std::chrono::milliseconds(200));
return mockExeNetworkVPU; }));
return mockExeNetworkOTHER; }));
ON_CALL(*core, compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
::testing::Matcher<const std::string&>(StrEq(CommonTestUtils::DEVICE_CPU)),
@ -112,9 +112,9 @@ public:
ON_CALL(*mockIExeNetGPU_1, get_property(StrEq(ov::optimal_number_of_infer_requests.name())))
.WillByDefault(Return(optimalNum));
inferReqInternalVPU = std::make_shared<NiceMock<ov::MockSyncInferRequest>>(mockIExeNetVPU);
mockExecutorVPU = std::make_shared<ov::threading::ImmediateExecutor>();
ON_CALL(*mockIExeNetVPU, get_property(StrEq(ov::optimal_number_of_infer_requests.name())))
inferReqInternalOTHER = std::make_shared<NiceMock<ov::MockSyncInferRequest>>(mockIExeNetOTHER);
mockExecutorOTHER = std::make_shared<ov::threading::ImmediateExecutor>();
ON_CALL(*mockIExeNetOTHER, get_property(StrEq(ov::optimal_number_of_infer_requests.name())))
.WillByDefault(Return(optimalNum));
}
};
@ -163,12 +163,13 @@ TEST_P(AutoRuntimeFallback, releaseResource) {
std::this_thread::sleep_for(std::chrono::milliseconds(0));
return mockInferrequestGPU_1; }));
}
} else if (deviceName == "VPU") {
mockInferrequestVPU = std::make_shared<ov::MockAsyncInferRequest>(
inferReqInternalVPU, mockExecutorVPU, nullptr, ifThrow);
ON_CALL(*mockIExeNetVPU.get(), create_infer_request()).WillByDefault(InvokeWithoutArgs([this]() {
} else if (deviceName == "OTHER") {
mockInferrequestOTHER =
std::make_shared<ov::MockAsyncInferRequest>(inferReqInternalOTHER, mockExecutorOTHER, nullptr, ifThrow);
ON_CALL(*mockIExeNetOTHER.get(), create_infer_request()).WillByDefault(InvokeWithoutArgs([this]() {
std::this_thread::sleep_for(std::chrono::milliseconds(0));
return mockInferrequestVPU; }));
return mockInferrequestOTHER;
}));
} else {
return;
}
@ -209,10 +210,10 @@ const std::vector<ConfigParams> testConfigs = {
ConfigParams{{{"GPU.0", false}, {"CPU", true}}, 2, true, false, false, false},
ConfigParams{{{"GPU.0", true}, {"CPU", true}}, 2, true, true, false, false},
// 3 devices
ConfigParams{{{"GPU.0", false}, {"GPU.1", false}, {"VPU", false}}, 1, true, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"VPU", false}}, 2, true, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"VPU", false}}, 3, true, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"VPU", true}}, 3, true, true, false, false},
ConfigParams{{{"GPU.0", false}, {"GPU.1", false}, {"OTHER", false}}, 1, true, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"OTHER", false}}, 2, true, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"OTHER", false}}, 3, true, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"OTHER", true}}, 3, true, true, false, false},
//CPU_HELP does not throw
ConfigParams{{{"GPU.0", false}, {"GPU.1", false}, {"CPU", false}}, 2, true, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"CPU", false}}, 2, true, false, false, false},
@ -233,10 +234,10 @@ const std::vector<ConfigParams> testConfigs = {
ConfigParams{{{"GPU.0", false}, {"CPU", true}}, 2, false, true, false, false},
ConfigParams{{{"GPU.0", true}, {"CPU", true}}, 2, false, true, false, false},
// 3 devices
ConfigParams{{{"GPU.0", false}, {"GPU.1", false}, {"VPU", false}}, 1, false, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"VPU", false}}, 1, false, true, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"VPU", false}}, 1, false, true, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"VPU", true}}, 1, false, true, false, false},
ConfigParams{{{"GPU.0", false}, {"GPU.1", false}, {"OTHER", false}}, 1, false, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"OTHER", false}}, 1, false, true, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"OTHER", false}}, 1, false, true, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"OTHER", true}}, 1, false, true, false, false},
//CPU_HELP does not throw
ConfigParams{{{"GPU.0", false}, {"GPU.1", false}, {"CPU", false}}, 2, false, false, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"CPU", false}}, 2, false, false, false, false},
@ -246,8 +247,8 @@ const std::vector<ConfigParams> testConfigs = {
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"CPU", true}}, 2, false, true, false, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", true}, {"CPU", true}}, 2, false, true, false, false},
// loadFail and CreateInferRequestFail
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"VPU", false}}, 3, true, false, true, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"VPU", false}}, 3, true, false, false, true},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"OTHER", false}}, 3, true, false, true, false},
ConfigParams{{{"GPU.0", true}, {"GPU.1", false}, {"OTHER", false}}, 3, true, false, false, true},
};
INSTANTIATE_TEST_SUITE_P(smoke_AutoRuntimeFallback, AutoRuntimeFallback,

211
src/plugins/auto/tests/unit/select_device_failed_test.cpp
View File

@ -173,7 +173,7 @@ TEST_P(AutoLoadFailedTest, LoadCNNetWork) {
// the test configure, for example
// ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
// DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, true},
// DeviceParams {"OTHER", true},
// DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 2, 3, 2},
//
// every element for ConfigParams
@ -188,63 +188,170 @@ TEST_P(AutoLoadFailedTest, LoadCNNetWork) {
// the inference request num is loadSuccessCount * optimalNum, in this test case optimalNum is 2
// so inference request num is 4 (CPU 2, MYRIAD 2)
//
const std::vector<ConfigParams> testConfigs = {ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, true},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, true},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 1, 2, 2},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, true},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 2, 3, 2},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, true},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 1, 2, 2},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, true},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, true},
DeviceParams {CommonTestUtils::DEVICE_CPU, false}}, 1, 2, 1},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, true},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, false}}, 1, 2, 1},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, true},
DeviceParams {CommonTestUtils::DEVICE_CPU, false}}, 2, 3, 1},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 3, 4, 2},
ConfigParams {false, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, false}}, 3, 4, 0},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, true},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 1, 2, 2},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 2, 3, 2},
ConfigParams {true, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, true},
DeviceParams {CommonTestUtils::DEVICE_CPU, false}}, 1, 2, 1},
ConfigParams {false, false, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, false}}, 2, 3, 0},
const std::vector<ConfigParams> testConfigs = {
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, true},
DeviceParams{"OTHER", true},
DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
1,
2,
2},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false},
DeviceParams{"OTHER", true},
DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
2,
3,
2},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, true},
DeviceParams{"OTHER", false},
DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
1,
2,
2},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, true},
DeviceParams{"OTHER", true},
DeviceParams{CommonTestUtils::DEVICE_CPU, false}},
1,
2,
1},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, true},
DeviceParams{"OTHER", false},
DeviceParams{CommonTestUtils::DEVICE_CPU, false}},
1,
2,
1},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false},
DeviceParams{"OTHER", true},
DeviceParams{CommonTestUtils::DEVICE_CPU, false}},
2,
3,
1},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false},
DeviceParams{"OTHER", false},
DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
3,
4,
2},
ConfigParams{false,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false},
DeviceParams{"OTHER", false},
DeviceParams{CommonTestUtils::DEVICE_CPU, false}},
3,
4,
0},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, true}, DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
1,
2,
2},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false}, DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
2,
3,
2},
ConfigParams{true,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, true}, DeviceParams{CommonTestUtils::DEVICE_CPU, false}},
1,
2,
1},
ConfigParams{false,
false,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false}, DeviceParams{CommonTestUtils::DEVICE_CPU, false}},
2,
3,
0},
ConfigParams{false, false, GENERAL, {DeviceParams{CommonTestUtils::DEVICE_GPU, false}}, 1, 1, 0},
ConfigParams{false, false, GENERAL, {DeviceParams{CommonTestUtils::DEVICE_CPU, false}}, 1, 1, 0},
ConfigParams{true, false, GENERAL, {DeviceParams{CommonTestUtils::DEVICE_GPU, true}}, 1, 1, 1},
ConfigParams{true, false, GENERAL, {DeviceParams{CommonTestUtils::DEVICE_CPU, true}}, 1, 1, 1},
ConfigParams{false, true, GENERAL, {DeviceParams{CommonTestUtils::DEVICE_GPU, true}}, 1, 0, 0},
ConfigParams{false, true, GENERAL, {DeviceParams{CommonTestUtils::DEVICE_CPU, true}}, 1, 0, 0},
ConfigParams {true, true, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, true},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 2, 2, 1},
ConfigParams {false, true, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, true},
DeviceParams {CommonTestUtils::DEVICE_CPU, false}}, 2, 2, 0},
ConfigParams {true, true, GENERAL, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 2, 2, 1},
ConfigParams {true, false, LATENCY, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 3, 3, 1},
ConfigParams {true, false, LATENCY, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 2, 2, 1},
ConfigParams {true, false, THROUGHPUT, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_KEEMBAY, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 3, 4, 2},
ConfigParams {true, false, THROUGHPUT, {DeviceParams {CommonTestUtils::DEVICE_GPU, false},
DeviceParams {CommonTestUtils::DEVICE_CPU, true}}, 2, 3, 2}
};
ConfigParams{true,
true,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false},
DeviceParams{"OTHER", true},
DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
2,
2,
1},
ConfigParams{false,
true,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false},
DeviceParams{"OTHER", true},
DeviceParams{CommonTestUtils::DEVICE_CPU, false}},
2,
2,
0},
ConfigParams{true,
true,
GENERAL,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false}, DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
2,
2,
1},
ConfigParams{true,
false,
LATENCY,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false},
DeviceParams{"OTHER", false},
DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
3,
3,
1},
ConfigParams{true,
false,
LATENCY,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false}, DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
2,
2,
1},
ConfigParams{true,
false,
THROUGHPUT,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false},
DeviceParams{"OTHER", false},
DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
3,
4,
2},
ConfigParams{true,
false,
THROUGHPUT,
{DeviceParams{CommonTestUtils::DEVICE_GPU, false}, DeviceParams{CommonTestUtils::DEVICE_CPU, true}},
2,
3,
2}};
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, AutoLoadFailedTest,
::testing::ValuesIn(testConfigs),

13
src/plugins/auto/tests/unit/select_device_test.cpp
View File

@ -17,11 +17,10 @@ using ConfigParams = std::tuple<
const DeviceInformation CPU_INFO = {CommonTestUtils::DEVICE_CPU, {}, 2, "01", "CPU_01"};
const DeviceInformation IGPU_INFO = {"GPU.0", {}, 2, "01", "iGPU_01"};
const DeviceInformation DGPU_INFO = {"GPU.1", {}, 2, "01", "dGPU_01"};
const DeviceInformation MYRIAD_INFO = {"MYRIAD", {}, 2, "01", "MYRIAD_01" };
const DeviceInformation KEEMBAY_INFO = {CommonTestUtils::DEVICE_KEEMBAY, {}, 2, "01", "VPU_01" };
const std::vector<DeviceInformation> fp32DeviceVector = {DGPU_INFO, IGPU_INFO, CPU_INFO, MYRIAD_INFO};
const std::vector<DeviceInformation> fp16DeviceVector = {DGPU_INFO, IGPU_INFO, MYRIAD_INFO, CPU_INFO};
const std::vector<DeviceInformation> int8DeviceVector = {KEEMBAY_INFO, DGPU_INFO, IGPU_INFO, CPU_INFO};
const DeviceInformation OTHERS_INFO = {"OTHERS", {}, 2, "01", "OTHERS" };
const std::vector<DeviceInformation> fp32DeviceVector = {DGPU_INFO, IGPU_INFO, OTHERS_INFO, CPU_INFO};
const std::vector<DeviceInformation> fp16DeviceVector = {DGPU_INFO, IGPU_INFO, OTHERS_INFO, CPU_INFO};
const std::vector<DeviceInformation> int8DeviceVector = {DGPU_INFO, IGPU_INFO, CPU_INFO};
const std::vector<DeviceInformation> binDeviceVector = {DGPU_INFO, IGPU_INFO, CPU_INFO};
const std::vector<DeviceInformation> batchedblobDeviceVector = {DGPU_INFO, IGPU_INFO};
std::map<std::string, const std::vector<DeviceInformation>> devicesMap = {{"FP32", fp32DeviceVector},
@ -30,8 +29,8 @@ std::map<std::string, const std::vector<DeviceInformation>> devicesMap = {{"FP32
{"BIN", binDeviceVector},
{"BATCHED_BLOB", batchedblobDeviceVector}
};
const std::vector<DeviceInformation> totalDevices = {DGPU_INFO, IGPU_INFO, MYRIAD_INFO, CPU_INFO, KEEMBAY_INFO};
const std::vector<DeviceInformation> reverseTotalDevices = {KEEMBAY_INFO, CPU_INFO, MYRIAD_INFO, IGPU_INFO, DGPU_INFO};
const std::vector<DeviceInformation> totalDevices = {DGPU_INFO, IGPU_INFO, OTHERS_INFO, CPU_INFO};
const std::vector<DeviceInformation> reverseTotalDevices = {CPU_INFO, OTHERS_INFO, IGPU_INFO, DGPU_INFO};
const std::vector<std::string> netPrecisions = {"FP32", "FP16", "INT8", "BIN", "BATCHED_BLOB"};
std::vector<ConfigParams> testConfigs;