Bell/execution device (#13737)
* debug only
Signed-off-by: fishbell <bell.song@intel.com>
* add execution_device property for auto
Signed-off-by: fishbell <bell.song@intel.com>
* Revert "debug only"
This reverts commit 5ef6140b63
.
* revert debug code
Signed-off-by: fishbell <bell.song@intel.com>
* polish tests to cover multiple gpu platform
Signed-off-by: fishbell <bell.song@intel.com>
clean up
Signed-off-by: fishbell <bell.song@intel.com>
* clang format
Signed-off-by: fishbell <bell.song@intel.com>
* only cpu in cpuFunctest
Signed-off-by: fishbell <bell.song@intel.com>
* thread-safe implementation
Signed-off-by: fishbell <bell.song@intel.com>
* remove not needed locl
Signed-off-by: fishbell <bell.song@intel.com>
* apply review comments
Signed-off-by: fishbell <bell.song@intel.com>
* remove blank line
Signed-off-by: fishbell <bell.song@intel.com>
Signed-off-by: fishbell <bell.song@intel.com>
This commit is contained in:
parent
57eddaa239
commit
394b6fe9ee
@ -1253,8 +1253,10 @@ int main(int argc, char* argv[]) {
|
||||
statistics->dump();
|
||||
|
||||
// Performance metrics report
|
||||
slog::info << "Count: " << iteration << " iterations" << slog::endl;
|
||||
slog::info << "Duration: " << double_to_string(totalDuration) << " ms" << slog::endl;
|
||||
if (device_name.find("AUTO") != std::string::npos)
|
||||
slog::info << "ExecutionDevice: " << compiledModel.get_property(ov::execution_devices) << slog::endl;
|
||||
slog::info << "Count: " << iteration << " iterations" << slog::endl;
|
||||
slog::info << "Duration: " << double_to_string(totalDuration) << " ms" << slog::endl;
|
||||
if (device_name.find("MULTI") == std::string::npos) {
|
||||
slog::info << "Latency:" << slog::endl;
|
||||
generalLatency.write_to_slog();
|
||||
|
@ -884,4 +884,10 @@ inline std::istream& operator>>(std::istream& is, Affinity& affinity) {
|
||||
* environment variable is set (as affinity is configured explicitly)
|
||||
*/
|
||||
static constexpr Property<Affinity> affinity{"AFFINITY"};
|
||||
|
||||
/**
|
||||
* @brief The devices that the inference task been executed.
|
||||
* @ingroup ov_runtime_cpp_prop_api
|
||||
*/
|
||||
static constexpr Property<std::string> execution_devices{"EXECUTION_DEVICES"};
|
||||
} // namespace ov
|
||||
|
@ -44,7 +44,8 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
|
||||
ov::PropertyName{ov::model_name.name(), ov::PropertyMutability::RO},
|
||||
ov::PropertyName{ov::optimal_number_of_infer_requests.name(), ov::PropertyMutability::RO},
|
||||
ov::PropertyName{ov::hint::model_priority.name(), ov::PropertyMutability::RO},
|
||||
ov::PropertyName{ov::device::priorities.name(), ov::PropertyMutability::RO}
|
||||
ov::PropertyName{ov::device::priorities.name(), ov::PropertyMutability::RO},
|
||||
ov::PropertyName{ov::execution_devices.name(), ov::PropertyMutability::RO}
|
||||
};
|
||||
} else if (name == ov::device::priorities) {
|
||||
auto value = _autoSContext->_config.find(ov::device::priorities.name());
|
||||
@ -143,6 +144,8 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
|
||||
}
|
||||
}
|
||||
return decltype(ov::optimal_number_of_infer_requests)::value_type {real};
|
||||
} else if (name == ov::execution_devices) {
|
||||
return decltype(ov::execution_devices)::value_type {_autoSContext->_exeDevices};
|
||||
}
|
||||
if (_autoSchedule->_loadContext[ACTUALDEVICE].isAlready) {
|
||||
return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetMetric(
|
||||
|
@ -159,7 +159,6 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
|
||||
deviceName += device.deviceName;
|
||||
deviceName += ((device.deviceName == validDevices.back().deviceName) ? "" : ",");
|
||||
}
|
||||
|
||||
_loadContext[ACTUALDEVICE].deviceInfo.deviceName = deviceName;
|
||||
_loadContext[ACTUALDEVICE].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] =
|
||||
InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT;
|
||||
@ -175,7 +174,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
|
||||
}
|
||||
LOG_INFO_TAG("select device:%s", _loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str());
|
||||
bool isActualDevCPU =
|
||||
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") !=std::string::npos;
|
||||
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") !=std::string::npos && !isCumulative;
|
||||
// if Actual device is CPU, disabled _loadContext[CPU], only use _loadContext[ACTUALDEVICE]
|
||||
if (isActualDevCPU || isCumulative) {
|
||||
_loadContext[CPU].isEnabled = false;
|
||||
@ -184,6 +183,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
|
||||
[=](const DeviceInformation& d) -> bool { return d.deviceName.find("CPU") != std::string::npos; });
|
||||
// if have CPU Device, enable _loadContext[CPU]
|
||||
if (CPUIter != _autoSContext->_devicePriorities.end()) {
|
||||
_autoSContext->_exeDevices = "(CPU)";
|
||||
_loadContext[CPU].isEnabled = true;
|
||||
_loadContext[CPU].deviceInfo = *CPUIter;
|
||||
_loadContext[CPU].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] =
|
||||
@ -314,6 +314,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
|
||||
_loadContext[CPU].executableNetwork._ptr.reset();
|
||||
_loadContext[CPU].executableNetwork._so.reset();
|
||||
LOG_INFO_TAG("helper released!!");
|
||||
_autoSContext->_exeDevices = _loadContext[ACTUALDEVICE].deviceInfo.deviceName;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -432,6 +433,10 @@ void AutoSchedule::WaitFirstNetworkReady() {
|
||||
// check if there is any device that have loaded network successfully
|
||||
for (int i = CONTEXTNUM - 1; i >= 0; i--) {
|
||||
if (_loadContext[i].isEnabled && _loadContext[i].isAlready) {
|
||||
if ( i != 0 ) {
|
||||
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
|
||||
_autoSContext->_exeDevices = _loadContext[i].deviceInfo.deviceName.substr(_loadContext[i].deviceInfo.deviceName.find(":") + 1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -441,6 +446,8 @@ void AutoSchedule::WaitFirstNetworkReady() {
|
||||
_loadContext[i].future.wait();
|
||||
// check if loading is successful
|
||||
if (_loadContext[i].isAlready) {
|
||||
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
|
||||
_autoSContext->_exeDevices = _loadContext[i].deviceInfo.deviceName.substr(_loadContext[i].deviceInfo.deviceName.find(":") + 1);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
@ -18,7 +18,7 @@
|
||||
#include "threading/ie_thread_safe_containers.hpp"
|
||||
#include "utils/log_util.hpp"
|
||||
#include <ie_performance_hints.hpp>
|
||||
#include "openvino/runtime/intel_auto/properties.hpp"
|
||||
#include "openvino/runtime/auto/properties.hpp"
|
||||
#include "ngraph/opsets/opset1.hpp"
|
||||
#include "transformations/utils/utils.hpp"
|
||||
#include "utils/log_util.hpp"
|
||||
@ -137,6 +137,7 @@ public:
|
||||
std::string _performanceHint;
|
||||
std::mutex _confMutex;
|
||||
MultiDeviceInferencePlugin* _plugin;
|
||||
std::string _exeDevices;
|
||||
virtual ~AutoScheduleContext() = default;
|
||||
};
|
||||
|
||||
|
@ -16,7 +16,7 @@
|
||||
#include <ie_metric_helpers.hpp>
|
||||
#include <ie_performance_hints.hpp>
|
||||
#include <threading/ie_executor_manager.hpp>
|
||||
#include "openvino/runtime/intel_auto/properties.hpp"
|
||||
#include "openvino/runtime/auto/properties.hpp"
|
||||
#include "plugin.hpp"
|
||||
#include <ie_algorithm.hpp>
|
||||
#include <ie_icore.hpp>
|
||||
|
@ -3,7 +3,7 @@
|
||||
//
|
||||
|
||||
#include "behavior/ov_plugin/properties_tests.hpp"
|
||||
#include <openvino/runtime/intel_auto/properties.hpp>
|
||||
#include <openvino/runtime/auto/properties.hpp>
|
||||
|
||||
using namespace ov::test::behavior;
|
||||
using namespace InferenceEngine::PluginConfigParams;
|
||||
@ -88,4 +88,14 @@ INSTANTIATE_TEST_SUITE_P(smoke_AutoBehaviorTests, OVPropertiesDefaultTests,
|
||||
::testing::Values(CommonTestUtils::DEVICE_AUTO),
|
||||
::testing::ValuesIn(default_properties)),
|
||||
OVPropertiesDefaultTests::getTestCaseName);
|
||||
|
||||
const std::vector<std::pair<ov::AnyMap, std::string>> autoExeDeviceConfigs = {
|
||||
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_CPU)}}, "CPU")
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_AutoMultiCompileModelBehaviorTests,
|
||||
OVCompileModelGetExecutionDeviceTests,
|
||||
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),
|
||||
::testing::ValuesIn(autoExeDeviceConfigs)),
|
||||
OVCompileModelGetExecutionDeviceTests::getTestCaseName);
|
||||
} // namespace
|
||||
|
@ -3,7 +3,7 @@
|
||||
//
|
||||
|
||||
#include "behavior/ov_plugin/properties_tests.hpp"
|
||||
#include <openvino/runtime/intel_auto/properties.hpp>
|
||||
#include <openvino/runtime/auto/properties.hpp>
|
||||
|
||||
using namespace ov::test::behavior;
|
||||
using namespace InferenceEngine::PluginConfigParams;
|
||||
@ -130,4 +130,23 @@ INSTANTIATE_TEST_SUITE_P(smoke_AutoMultiCompileModelBehaviorTests,
|
||||
::testing::ValuesIn(auto_multi_plugin_properties),
|
||||
::testing::ValuesIn(auto_multi_compileModel_properties)),
|
||||
OVSetPropComplieModleGetPropTests::getTestCaseName);
|
||||
|
||||
const std::vector<std::pair<ov::AnyMap, std::string>> autoExeDeviceConfigs = {
|
||||
std::make_pair(ov::AnyMap{{ov::device::priorities("GPU.0")}}, "GPU.0"),
|
||||
#ifdef ENABLE_INTEL_CPU
|
||||
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_GPU, CommonTestUtils::DEVICE_CPU)}}, "undefined"),
|
||||
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_CPU, CommonTestUtils::DEVICE_GPU)}}, "CPU"),
|
||||
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_CPU, CommonTestUtils::DEVICE_GPU),
|
||||
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}}, "CPU,GPU"),
|
||||
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_GPU, CommonTestUtils::DEVICE_CPU),
|
||||
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}}, "GPU,CPU"),
|
||||
#endif
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_AutoMultiCompileModelBehaviorTests,
|
||||
OVCompileModelGetExecutionDeviceTests,
|
||||
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),
|
||||
::testing::ValuesIn(autoExeDeviceConfigs)),
|
||||
OVCompileModelGetExecutionDeviceTests::getTestCaseName);
|
||||
|
||||
} // namespace
|
||||
|
@ -5,7 +5,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "base/ov_behavior_test_utils.hpp"
|
||||
#include "openvino/runtime/intel_auto/properties.hpp"
|
||||
#include "openvino/runtime/auto/properties.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace test {
|
||||
|
@ -58,6 +58,22 @@ public:
|
||||
};
|
||||
using OVSetPropComplieModleWihtIncorrectPropTests = OVSetPropComplieModleGetPropTests;
|
||||
|
||||
using OvPropertiesParams = std::tuple<
|
||||
std::string, // device name
|
||||
std::pair<ov::AnyMap, std::string> // device and expect execution device configuration
|
||||
>;
|
||||
class OVCompileModelGetExecutionDeviceTests : public testing::WithParamInterface<OvPropertiesParams>,
|
||||
public OVPropertiesBase {
|
||||
public:
|
||||
static std::string getTestCaseName(testing::TestParamInfo<OvPropertiesParams> obj);
|
||||
|
||||
void SetUp() override;
|
||||
|
||||
AnyMap compileModelProperties;
|
||||
|
||||
std::string expectedDeviceName;
|
||||
};
|
||||
|
||||
} // namespace behavior
|
||||
} // namespace test
|
||||
} // namespace ov
|
@ -154,6 +154,56 @@ TEST_P(OVSetPropComplieModleWihtIncorrectPropTests, CanNotCompileModelWithIncorr
|
||||
ASSERT_THROW(core->compile_model(model, target_device, properties), ov::Exception);
|
||||
}
|
||||
|
||||
std::string OVCompileModelGetExecutionDeviceTests::getTestCaseName(testing::TestParamInfo<OvPropertiesParams> obj) {
|
||||
std::string target_device;
|
||||
std::pair<ov::AnyMap, std::string> userConfig;
|
||||
std::tie(target_device, userConfig) = obj.param;
|
||||
std::replace(target_device.begin(), target_device.end(), ':', '.');
|
||||
auto compileModelProperties = userConfig.first;
|
||||
std::ostringstream result;
|
||||
result << "device_name=" << target_device << "_";
|
||||
if (!compileModelProperties.empty()) {
|
||||
result << "_compileModelProp=" << util::join(util::split(util::to_string(compileModelProperties), ' '), "_");
|
||||
}
|
||||
result << "_expectedDevice=" << userConfig.second;
|
||||
return result.str();
|
||||
}
|
||||
|
||||
void OVCompileModelGetExecutionDeviceTests::SetUp() {
|
||||
SKIP_IF_CURRENT_TEST_IS_DISABLED();
|
||||
std::pair<ov::AnyMap, std::string> userConfig;
|
||||
std::tie(target_device, userConfig) = GetParam();
|
||||
compileModelProperties = userConfig.first;
|
||||
expectedDeviceName = userConfig.second;
|
||||
model = ngraph::builder::subgraph::makeConvPoolRelu();
|
||||
}
|
||||
|
||||
TEST_P(OVCompileModelGetExecutionDeviceTests, CanGetExecutionDeviceInfo) {
|
||||
ov::CompiledModel exeNetWork;
|
||||
auto deviceList = core->get_available_devices();
|
||||
std::string updatedExpectDevices = expectedDeviceName;
|
||||
for (auto &iter : compileModelProperties) {
|
||||
if (iter.first == ov::hint::performance_mode && iter.second.as<ov::hint::PerformanceMode>() == ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT) {
|
||||
std::vector<std::string> expected_devices = util::split(expectedDeviceName, ',');
|
||||
std::vector<std::string> sameTypeDevices;
|
||||
for (auto& deviceName : expected_devices) {
|
||||
for (auto&& device : deviceList) {
|
||||
if (device.find(deviceName) != std::string::npos) {
|
||||
sameTypeDevices.push_back(std::move(device));
|
||||
}
|
||||
}
|
||||
}
|
||||
updatedExpectDevices = util::join(sameTypeDevices, ",");
|
||||
}
|
||||
}
|
||||
OV_ASSERT_NO_THROW(exeNetWork = core->compile_model(model, target_device, compileModelProperties));
|
||||
ov::Any property;
|
||||
OV_ASSERT_NO_THROW(property = exeNetWork.get_property(ov::execution_devices));
|
||||
if (expectedDeviceName.find("undefined") == std::string::npos)
|
||||
ASSERT_EQ(property, updatedExpectDevices);
|
||||
else
|
||||
ASSERT_FALSE(property.empty());
|
||||
}
|
||||
} // namespace behavior
|
||||
} // namespace test
|
||||
} // namespace ov
|
||||
|
@ -575,26 +575,27 @@ def main():
|
||||
])
|
||||
statistics.dump()
|
||||
|
||||
|
||||
print(f'Count: {iteration} iterations')
|
||||
print(f'Duration: {get_duration_in_milliseconds(total_duration_sec):.2f} ms')
|
||||
if devices.count("AUTO"):
|
||||
print(f'ExecutionDevice: {compiled_model.get_property("EXECUTION_DEVICES")}')
|
||||
print(f'Count: {iteration} iterations')
|
||||
print(f'Duration: {get_duration_in_milliseconds(total_duration_sec):.2f} ms')
|
||||
if MULTI_DEVICE_NAME not in device_name:
|
||||
print('Latency:')
|
||||
if args.latency_percentile == 50 and static_mode:
|
||||
print(f' Median: {median_latency_ms:.2f} ms')
|
||||
print(f' Median: {median_latency_ms:.2f} ms')
|
||||
elif args.latency_percentile != 50:
|
||||
print(f'({args.latency_percentile} percentile): {median_latency_ms:.2f} ms')
|
||||
print(f' AVG: {avg_latency_ms:.2f} ms')
|
||||
print(f' MIN: {min_latency_ms:.2f} ms')
|
||||
print(f' MAX: {max_latency_ms:.2f} ms')
|
||||
print(f' AVG: {avg_latency_ms:.2f} ms')
|
||||
print(f' MIN: {min_latency_ms:.2f} ms')
|
||||
print(f' MAX: {max_latency_ms:.2f} ms')
|
||||
|
||||
if pcseq:
|
||||
print("Latency for each data shape group: ")
|
||||
for group in benchmark.latency_groups:
|
||||
print(f" {str(group)}")
|
||||
print(f' AVG: {group.avg:.2f} ms')
|
||||
print(f' MIN: {group.min:.2f} ms')
|
||||
print(f' MAX: {group.max:.2f} ms')
|
||||
print(f' AVG: {group.avg:.2f} ms')
|
||||
print(f' MIN: {group.min:.2f} ms')
|
||||
print(f' MAX: {group.max:.2f} ms')
|
||||
|
||||
print(f'Throughput: {fps:.2f} FPS')
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user