Bell/execution device (#13737)

* debug only

Signed-off-by: fishbell <bell.song@intel.com>

* add execution_device property for auto

Signed-off-by: fishbell <bell.song@intel.com>

* Revert "debug only"

This reverts commit 5ef6140b63.

* revert debug code

Signed-off-by: fishbell <bell.song@intel.com>

* polish tests to cover multiple gpu platform

Signed-off-by: fishbell <bell.song@intel.com>

clean up

Signed-off-by: fishbell <bell.song@intel.com>

* clang format

Signed-off-by: fishbell <bell.song@intel.com>

* only cpu in cpuFunctest

Signed-off-by: fishbell <bell.song@intel.com>

* thread-safe implementation

Signed-off-by: fishbell <bell.song@intel.com>

* remove not needed locl

Signed-off-by: fishbell <bell.song@intel.com>

* apply review comments

Signed-off-by: fishbell <bell.song@intel.com>

* remove blank line

Signed-off-by: fishbell <bell.song@intel.com>

Signed-off-by: fishbell <bell.song@intel.com>
This commit is contained in:
yanlan song 2022-11-07 10:12:28 +08:00 committed by GitHub
parent 57eddaa239
commit 394b6fe9ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 135 additions and 20 deletions

View File

@ -1253,8 +1253,10 @@ int main(int argc, char* argv[]) {
statistics->dump();
// Performance metrics report
slog::info << "Count: " << iteration << " iterations" << slog::endl;
slog::info << "Duration: " << double_to_string(totalDuration) << " ms" << slog::endl;
if (device_name.find("AUTO") != std::string::npos)
slog::info << "ExecutionDevice: " << compiledModel.get_property(ov::execution_devices) << slog::endl;
slog::info << "Count: " << iteration << " iterations" << slog::endl;
slog::info << "Duration: " << double_to_string(totalDuration) << " ms" << slog::endl;
if (device_name.find("MULTI") == std::string::npos) {
slog::info << "Latency:" << slog::endl;
generalLatency.write_to_slog();

View File

@ -884,4 +884,10 @@ inline std::istream& operator>>(std::istream& is, Affinity& affinity) {
* environment variable is set (as affinity is configured explicitly)
*/
static constexpr Property<Affinity> affinity{"AFFINITY"};
/**
* @brief The devices that the inference task been executed.
* @ingroup ov_runtime_cpp_prop_api
*/
static constexpr Property<std::string> execution_devices{"EXECUTION_DEVICES"};
} // namespace ov

View File

@ -44,7 +44,8 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
ov::PropertyName{ov::model_name.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::optimal_number_of_infer_requests.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::hint::model_priority.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::device::priorities.name(), ov::PropertyMutability::RO}
ov::PropertyName{ov::device::priorities.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::execution_devices.name(), ov::PropertyMutability::RO}
};
} else if (name == ov::device::priorities) {
auto value = _autoSContext->_config.find(ov::device::priorities.name());
@ -143,6 +144,8 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
}
}
return decltype(ov::optimal_number_of_infer_requests)::value_type {real};
} else if (name == ov::execution_devices) {
return decltype(ov::execution_devices)::value_type {_autoSContext->_exeDevices};
}
if (_autoSchedule->_loadContext[ACTUALDEVICE].isAlready) {
return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetMetric(

View File

@ -159,7 +159,6 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
deviceName += device.deviceName;
deviceName += ((device.deviceName == validDevices.back().deviceName) ? "" : ",");
}
_loadContext[ACTUALDEVICE].deviceInfo.deviceName = deviceName;
_loadContext[ACTUALDEVICE].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] =
InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT;
@ -175,7 +174,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
}
LOG_INFO_TAG("select device:%s", _loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str());
bool isActualDevCPU =
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") !=std::string::npos;
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") !=std::string::npos && !isCumulative;
// if Actual device is CPU, disabled _loadContext[CPU], only use _loadContext[ACTUALDEVICE]
if (isActualDevCPU || isCumulative) {
_loadContext[CPU].isEnabled = false;
@ -184,6 +183,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
[=](const DeviceInformation& d) -> bool { return d.deviceName.find("CPU") != std::string::npos; });
// if have CPU Device, enable _loadContext[CPU]
if (CPUIter != _autoSContext->_devicePriorities.end()) {
_autoSContext->_exeDevices = "(CPU)";
_loadContext[CPU].isEnabled = true;
_loadContext[CPU].deviceInfo = *CPUIter;
_loadContext[CPU].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] =
@ -314,6 +314,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
_loadContext[CPU].executableNetwork._ptr.reset();
_loadContext[CPU].executableNetwork._so.reset();
LOG_INFO_TAG("helper released!!");
_autoSContext->_exeDevices = _loadContext[ACTUALDEVICE].deviceInfo.deviceName;
break;
}
}
@ -432,6 +433,10 @@ void AutoSchedule::WaitFirstNetworkReady() {
// check if there is any device that have loaded network successfully
for (int i = CONTEXTNUM - 1; i >= 0; i--) {
if (_loadContext[i].isEnabled && _loadContext[i].isAlready) {
if ( i != 0 ) {
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
_autoSContext->_exeDevices = _loadContext[i].deviceInfo.deviceName.substr(_loadContext[i].deviceInfo.deviceName.find(":") + 1);
}
return;
}
}
@ -441,6 +446,8 @@ void AutoSchedule::WaitFirstNetworkReady() {
_loadContext[i].future.wait();
// check if loading is successful
if (_loadContext[i].isAlready) {
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
_autoSContext->_exeDevices = _loadContext[i].deviceInfo.deviceName.substr(_loadContext[i].deviceInfo.deviceName.find(":") + 1);
return;
}
}

View File

@ -18,7 +18,7 @@
#include "threading/ie_thread_safe_containers.hpp"
#include "utils/log_util.hpp"
#include <ie_performance_hints.hpp>
#include "openvino/runtime/intel_auto/properties.hpp"
#include "openvino/runtime/auto/properties.hpp"
#include "ngraph/opsets/opset1.hpp"
#include "transformations/utils/utils.hpp"
#include "utils/log_util.hpp"
@ -137,6 +137,7 @@ public:
std::string _performanceHint;
std::mutex _confMutex;
MultiDeviceInferencePlugin* _plugin;
std::string _exeDevices;
virtual ~AutoScheduleContext() = default;
};

View File

@ -16,7 +16,7 @@
#include <ie_metric_helpers.hpp>
#include <ie_performance_hints.hpp>
#include <threading/ie_executor_manager.hpp>
#include "openvino/runtime/intel_auto/properties.hpp"
#include "openvino/runtime/auto/properties.hpp"
#include "plugin.hpp"
#include <ie_algorithm.hpp>
#include <ie_icore.hpp>

View File

@ -3,7 +3,7 @@
//
#include "behavior/ov_plugin/properties_tests.hpp"
#include <openvino/runtime/intel_auto/properties.hpp>
#include <openvino/runtime/auto/properties.hpp>
using namespace ov::test::behavior;
using namespace InferenceEngine::PluginConfigParams;
@ -88,4 +88,14 @@ INSTANTIATE_TEST_SUITE_P(smoke_AutoBehaviorTests, OVPropertiesDefaultTests,
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(default_properties)),
OVPropertiesDefaultTests::getTestCaseName);
const std::vector<std::pair<ov::AnyMap, std::string>> autoExeDeviceConfigs = {
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_CPU)}}, "CPU")
};
INSTANTIATE_TEST_SUITE_P(smoke_AutoMultiCompileModelBehaviorTests,
OVCompileModelGetExecutionDeviceTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(autoExeDeviceConfigs)),
OVCompileModelGetExecutionDeviceTests::getTestCaseName);
} // namespace

View File

@ -3,7 +3,7 @@
//
#include "behavior/ov_plugin/properties_tests.hpp"
#include <openvino/runtime/intel_auto/properties.hpp>
#include <openvino/runtime/auto/properties.hpp>
using namespace ov::test::behavior;
using namespace InferenceEngine::PluginConfigParams;
@ -130,4 +130,23 @@ INSTANTIATE_TEST_SUITE_P(smoke_AutoMultiCompileModelBehaviorTests,
::testing::ValuesIn(auto_multi_plugin_properties),
::testing::ValuesIn(auto_multi_compileModel_properties)),
OVSetPropComplieModleGetPropTests::getTestCaseName);
const std::vector<std::pair<ov::AnyMap, std::string>> autoExeDeviceConfigs = {
std::make_pair(ov::AnyMap{{ov::device::priorities("GPU.0")}}, "GPU.0"),
#ifdef ENABLE_INTEL_CPU
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_GPU, CommonTestUtils::DEVICE_CPU)}}, "undefined"),
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_CPU, CommonTestUtils::DEVICE_GPU)}}, "CPU"),
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_CPU, CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}}, "CPU,GPU"),
std::make_pair(ov::AnyMap{{ov::device::priorities(CommonTestUtils::DEVICE_GPU, CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}}, "GPU,CPU"),
#endif
};
INSTANTIATE_TEST_SUITE_P(smoke_AutoMultiCompileModelBehaviorTests,
OVCompileModelGetExecutionDeviceTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(autoExeDeviceConfigs)),
OVCompileModelGetExecutionDeviceTests::getTestCaseName);
} // namespace

View File

@ -5,7 +5,7 @@
#pragma once
#include "base/ov_behavior_test_utils.hpp"
#include "openvino/runtime/intel_auto/properties.hpp"
#include "openvino/runtime/auto/properties.hpp"
namespace ov {
namespace test {

View File

@ -58,6 +58,22 @@ public:
};
using OVSetPropComplieModleWihtIncorrectPropTests = OVSetPropComplieModleGetPropTests;
using OvPropertiesParams = std::tuple<
std::string, // device name
std::pair<ov::AnyMap, std::string> // device and expect execution device configuration
>;
class OVCompileModelGetExecutionDeviceTests : public testing::WithParamInterface<OvPropertiesParams>,
public OVPropertiesBase {
public:
static std::string getTestCaseName(testing::TestParamInfo<OvPropertiesParams> obj);
void SetUp() override;
AnyMap compileModelProperties;
std::string expectedDeviceName;
};
} // namespace behavior
} // namespace test
} // namespace ov

View File

@ -154,6 +154,56 @@ TEST_P(OVSetPropComplieModleWihtIncorrectPropTests, CanNotCompileModelWithIncorr
ASSERT_THROW(core->compile_model(model, target_device, properties), ov::Exception);
}
std::string OVCompileModelGetExecutionDeviceTests::getTestCaseName(testing::TestParamInfo<OvPropertiesParams> obj) {
std::string target_device;
std::pair<ov::AnyMap, std::string> userConfig;
std::tie(target_device, userConfig) = obj.param;
std::replace(target_device.begin(), target_device.end(), ':', '.');
auto compileModelProperties = userConfig.first;
std::ostringstream result;
result << "device_name=" << target_device << "_";
if (!compileModelProperties.empty()) {
result << "_compileModelProp=" << util::join(util::split(util::to_string(compileModelProperties), ' '), "_");
}
result << "_expectedDevice=" << userConfig.second;
return result.str();
}
void OVCompileModelGetExecutionDeviceTests::SetUp() {
SKIP_IF_CURRENT_TEST_IS_DISABLED();
std::pair<ov::AnyMap, std::string> userConfig;
std::tie(target_device, userConfig) = GetParam();
compileModelProperties = userConfig.first;
expectedDeviceName = userConfig.second;
model = ngraph::builder::subgraph::makeConvPoolRelu();
}
TEST_P(OVCompileModelGetExecutionDeviceTests, CanGetExecutionDeviceInfo) {
ov::CompiledModel exeNetWork;
auto deviceList = core->get_available_devices();
std::string updatedExpectDevices = expectedDeviceName;
for (auto &iter : compileModelProperties) {
if (iter.first == ov::hint::performance_mode && iter.second.as<ov::hint::PerformanceMode>() == ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT) {
std::vector<std::string> expected_devices = util::split(expectedDeviceName, ',');
std::vector<std::string> sameTypeDevices;
for (auto& deviceName : expected_devices) {
for (auto&& device : deviceList) {
if (device.find(deviceName) != std::string::npos) {
sameTypeDevices.push_back(std::move(device));
}
}
}
updatedExpectDevices = util::join(sameTypeDevices, ",");
}
}
OV_ASSERT_NO_THROW(exeNetWork = core->compile_model(model, target_device, compileModelProperties));
ov::Any property;
OV_ASSERT_NO_THROW(property = exeNetWork.get_property(ov::execution_devices));
if (expectedDeviceName.find("undefined") == std::string::npos)
ASSERT_EQ(property, updatedExpectDevices);
else
ASSERT_FALSE(property.empty());
}
} // namespace behavior
} // namespace test
} // namespace ov

View File

@ -575,26 +575,27 @@ def main():
])
statistics.dump()
print(f'Count: {iteration} iterations')
print(f'Duration: {get_duration_in_milliseconds(total_duration_sec):.2f} ms')
if devices.count("AUTO"):
print(f'ExecutionDevice: {compiled_model.get_property("EXECUTION_DEVICES")}')
print(f'Count: {iteration} iterations')
print(f'Duration: {get_duration_in_milliseconds(total_duration_sec):.2f} ms')
if MULTI_DEVICE_NAME not in device_name:
print('Latency:')
if args.latency_percentile == 50 and static_mode:
print(f' Median: {median_latency_ms:.2f} ms')
print(f' Median: {median_latency_ms:.2f} ms')
elif args.latency_percentile != 50:
print(f'({args.latency_percentile} percentile): {median_latency_ms:.2f} ms')
print(f' AVG: {avg_latency_ms:.2f} ms')
print(f' MIN: {min_latency_ms:.2f} ms')
print(f' MAX: {max_latency_ms:.2f} ms')
print(f' AVG: {avg_latency_ms:.2f} ms')
print(f' MIN: {min_latency_ms:.2f} ms')
print(f' MAX: {max_latency_ms:.2f} ms')
if pcseq:
print("Latency for each data shape group: ")
for group in benchmark.latency_groups:
print(f" {str(group)}")
print(f' AVG: {group.avg:.2f} ms')
print(f' MIN: {group.min:.2f} ms')
print(f' MAX: {group.max:.2f} ms')
print(f' AVG: {group.avg:.2f} ms')
print(f' MIN: {group.min:.2f} ms')
print(f' MAX: {group.max:.2f} ms')
print(f'Throughput: {fps:.2f} FPS')