[GPU] Add new properties and fix bechmark_app (#10149)

This commit is contained in:
Sergey Shlyapnikov authored and GitHub committed 2022-02-09 09:18:54 +03:00
1 parent 1970baeb1c
commit 8f0e974ee6
19 files changed
+706 -208

No files matched your search

+41 -16
View File
@@ -278,18 +278,26 @@ int main(int argc, char* argv[]) {
// the rest are individual per-device settings (overriding the values set with perf modes)
auto setThroughputStreams = [&]() {
const std::string key = getDeviceTypeFromName(device) + "_THROUGHPUT_STREAMS";
std::string key = getDeviceTypeFromName(device) + "_THROUGHPUT_STREAMS";
if (device_nstreams.count(device)) {
// set to user defined value
auto supported_properties = core.get_property(device, ov::supported_properties);
if (std::find(supported_properties.begin(), supported_properties.end(), key) ==
if (std::find(supported_properties.begin(), supported_properties.end(), key) !=
supported_properties.end()) {
throw std::logic_error("Device " + device + " doesn't support config key '" + key + "'! " +
device_config[key] = device_nstreams.at(device);
} else if (std::find(supported_properties.begin(),
supported_properties.end(),
ov::streams::num.name()) != supported_properties.end()) {
// Use API 2.0 key for streams
key = ov::streams::num.name();
device_config[key] = device_nstreams.at(device);
} else {
throw std::logic_error("Device " + device + " doesn't support config key '" + key + "' " +
"and '" + ov::streams::num.name() + "'!" +
"Please specify -nstreams for correct devices in format "
"<dev1>:<nstreams1>,<dev2>:<nstreams2>" +
" or via configuration file.");
}
device_config[key] = device_nstreams.at(device);
} else if (ov_perf_hint == ov::hint::PerformanceMode::UNDEFINED && !device_config.count(key) &&
(FLAGS_api == "async")) {
slog::warn << "-nstreams default value is determined automatically for " << device
@@ -299,9 +307,20 @@ int main(int argc, char* argv[]) {
"but it still may be non-optimal for some cases, for more "
"information look at README."
<< slog::endl;
if (std::string::npos == device.find("MYRIAD")) // MYRIAD sets the default number of
// streams implicitly (without _AUTO)
device_config[key] = std::string(getDeviceTypeFromName(device) + "_THROUGHPUT_AUTO");
if (std::string::npos == device.find("MYRIAD")) { // MYRIAD sets the default number of
// streams implicitly (without _AUTO)
auto supported_properties = core.get_property(device, ov::supported_properties);
if (std::find(supported_properties.begin(), supported_properties.end(), key) !=
supported_properties.end()) {
device_config[key] = std::string(getDeviceTypeFromName(device) + "_THROUGHPUT_AUTO");
} else if (std::find(supported_properties.begin(),
supported_properties.end(),
ov::streams::num.name()) != supported_properties.end()) {
// Use API 2.0 key for streams
key = ov::streams::num.name();
device_config[key] = std::to_string(ov::streams::AUTO);
}
}
}
if (device_config.count(key))
device_nstreams[device] = device_config.at(key).as<std::string>();
@@ -630,22 +649,28 @@ int main(int argc, char* argv[]) {
next_step();
// output of the actual settings that the device selected
for (const auto& device : devices) {
auto supported_properties = compiledModel.get_property(ov::supported_properties);
auto supported_properties = core.get_property(device, ov::supported_properties);
slog::info << "Device: " << device << slog::endl;
for (const auto& cfg : supported_properties) {
slog::info << " {" << cfg << " , ";
std::stringstream strm;
compiledModel.get_property(cfg).print(strm);
strm << "";
slog::info << strm.str();
slog::info << " }" << slog::endl;
try {
if (cfg == ov::supported_properties)
continue;
auto prop = compiledModel.get_property(cfg);
slog::info << " { " << cfg << " , " << prop.as<std::string>() << " }" << slog::endl;
} catch (const ov::Exception&) {
}
}
}
// Update number of streams
for (auto&& ds : device_nstreams) {
const std::string key = getDeviceTypeFromName(ds.first) + "_THROUGHPUT_STREAMS";
device_nstreams[ds.first] = core.get_property(ds.first, key).as<std::string>();
try {
const std::string key = getDeviceTypeFromName(ds.first) + "_THROUGHPUT_STREAMS";
device_nstreams[ds.first] = core.get_property(ds.first, key).as<std::string>();
} catch (const ov::Exception&) {
device_nstreams[ds.first] = core.get_property(ds.first, ov::streams::num.name()).as<std::string>();
}
}
// Number of requests
+21 -2
View File
@@ -71,14 +71,33 @@ void print_any_value(const ov::Any& value) {
slog::info << std::get<1>(values);
slog::info << " }";
slog::info << slog::endl;
} else if (value.is<std::map<ov::element::Type, float>>()) {
auto values = value.as<std::map<ov::element::Type, float>>();
slog::info << "{ ";
for (auto& kv : values) {
slog::info << kv.first << ": " << kv.second << "; ";
}
slog::info << " }";
slog::info << slog::endl;
} else if (value.is<std::map<std::string, uint64_t>>()) {
auto values = value.as<std::map<std::string, uint64_t>>();
slog::info << "{ ";
for (auto& kv : values) {
slog::info << kv.first << ": " << kv.second << "; ";
}
slog::info << " }";
slog::info << slog::endl;
} else if (value.is<ov::hint::PerformanceMode>()) {
auto values = value.as<std::string>();
slog::info << (values.empty() ? "\"\"" : values) << slog::endl;
} else {
std::stringstream strm;
value.print(strm);
auto str = strm.str();
if (str.empty()) {
std::cout << "UNSUPPORTED TYPE" << std::endl;
slog::info << "UNSUPPORTED TYPE" << slog::endl;
} else {
std::cout << str << std::endl;
slog::info << str << slog::endl;
}
}
}
@@ -26,9 +26,9 @@ def main():
log.info('Available devices:')
for device in core.available_devices:
log.info(f'{device} :')
log.info('\tSUPPORTED_METRICS:')
for property_key in core.get_property(device, 'SUPPORTED_METRICS'):
if property_key not in ('SUPPORTED_METRICS', 'SUPPORTED_CONFIG_KEYS'):
log.info('\tSUPPORTED_PROPERTIES:')
for property_key in core.get_property(device, 'SUPPORTED_PROPERTIES'):
if property_key not in ('SUPPORTED_METRICS', 'SUPPORTED_CONFIG_KEYS', 'SUPPORTED_PROPERTIES'):
try:
property_val = core.get_property(device, property_key)
except TypeError:
@@ -36,15 +36,6 @@ def main():
log.info(f'\t\t{property_key}: {param_to_string(property_val)}')
log.info('')
log.info('\tSUPPORTED_CONFIG_KEYS (default values):')
for config_key in core.get_property(device, 'SUPPORTED_CONFIG_KEYS'):
try:
config_val = core.get_property(device, config_key)
except TypeError:
config_val = 'UNSUPPORTED TYPE'
log.info(f'\t\t{config_key}: {param_to_string(config_val)}')
log.info('')
# -----------------------------------------------------------------------------------------------------------------
return 0
@@ -293,6 +293,17 @@ PyAny from_ov_any(const ov::Any& any) {
PyDict_SetItemString(dict, it.first.c_str(), PyLong_FromLong((long)it.second));
}
return dict;
}
// Check for std::vector<ov::PropertyName>
else if (any.is<std::vector<ov::PropertyName>>()) {
auto val = any.as<std::vector<ov::PropertyName>>();
PyObject* dict = PyDict_New();
for (const auto& it : val) {
std::string property_name = it;
std::string mutability = it.is_mutable() ? "RW" : "RO";
PyDict_SetItemString(dict, property_name.c_str(), PyUnicode_FromString(mutability.c_str()));
}
return dict;
} else {
PyErr_SetString(PyExc_TypeError, "Failed to convert parameter to Python representation!");
return (PyObject*)NULL;
@@ -8,8 +8,6 @@
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <ie_parameter.hpp>
#include <ie_plugin_config.hpp>
#include <openvino/core/type/element_type.hpp>
#include <string>
@@ -18,6 +16,7 @@
#include "openvino/runtime/compiled_model.hpp"
#include "openvino/runtime/infer_request.hpp"
#include "openvino/runtime/tensor.hpp"
#include "openvino/runtime/properties.hpp"
#include "pyopenvino/core/containers.hpp"
#include "pyopenvino/graph/any.hpp"
@@ -10,9 +10,31 @@
#pragma once
#include "ie_plugin_config.hpp"
#include "openvino/runtime/properties.hpp"
namespace InferenceEngine {
/**
* @brief A namespace with helper functions
* @ingroup ie_dev_api_plugin_api
*/
namespace util {
template <typename T, ov::PropertyMutability M>
inline T string_to_property(const std::string& val, const ov::util::BaseProperty<T, M>& property) {
std::stringstream ss(val);
T value;
ss >> value;
return value;
}
template <typename T>
inline std::string property_to_string(const T& property) {
std::stringstream ss;
ss << property;
return ss.str();
}
} // namespace util
/**
* @brief A namespace with internal plugin configuration keys
* @ingroup ie_dev_api_plugin_api
@@ -61,7 +61,7 @@ using ThrottleLevel = ov::hint::Priority;
* as defined in https://www.khronos.org/registry/OpenCL/specs/opencl-2.1-extensions.pdf,
* chapter 9.19. This option should be used with ov::intl_gpu::hint::ThrottleLevel values.
*/
static constexpr Property<ov::hint::Priority> queue_throttle{"GPU_QUEUE_THROTTLE"};
static constexpr Property<ThrottleLevel> queue_throttle{"GPU_QUEUE_THROTTLE"};
/**
* @brief This key instructs the GPU plugin to use the OpenCL queue priority hint
@@ -80,7 +80,7 @@ static constexpr Property<ov::hint::Priority> queue_priority{"GPU_QUEUE_PRIORITY
* - MEDIUM (DEFAULT) - instructs the GPU Plugin to use any available cores (BIG or LITTLE cores)
* - HIGH - instructs the GPU Plugin to use BIG cores if they are available
*/
static constexpr Property<ov::hint::Priority> host_task_priority{"OV_GPU_HOST_TASK_PRIORITY"};
static constexpr Property<ov::hint::Priority> host_task_priority{"GPU_HOST_TASK_PRIORITY"};
/**
* @brief This key identifies available device memory size in bytes
@@ -121,6 +121,7 @@ struct BaseProperty : public PropertyTag {
private:
const char* _name = nullptr;
};
template <typename T, PropertyMutability M>
inline std::ostream& operator<<(std::ostream& os, const BaseProperty<T, M>& property) {
return os << property.name();
@@ -232,7 +233,7 @@ inline std::istream& operator>>(std::istream& is, Priority& priority) {
* @brief High-level OpenVINO model priority hint
* Defines what model should be provided with more performant bounded resource first
*/
static constexpr Property<Priority> model_priority{"OV_MODEL_PRIORITY"};
static constexpr Property<Priority> model_priority{"MODEL_PRIORITY"};
/**
* @brief Enum to define possible performance mode hints
@@ -292,6 +293,11 @@ static constexpr Property<uint32_t> num_requests{"PERFORMANCE_HINT_NUM_REQUESTS"
* ov::optimal_batch_size)
*/
static constexpr Property<std::shared_ptr<ov::Model>> model{"MODEL_PTR"};
/**
* @brief Special key for auto batching feature configuration. Enabled by default
*/
static constexpr Property<bool, PropertyMutability::RW> allow_auto_batching{"ALLOW_AUTO_BATCHING"};
} // namespace hint
/**
+1 -1
View File
@@ -544,7 +544,7 @@ public:
deviceNameWithoutBatch = DeviceIDParser::getBatchDevice(deviceNameWithBatchSize);
} else {
// check whether the Auto-Batching is disabled explicitly
const auto& batch_mode = config.find(CONFIG_KEY(ALLOW_AUTO_BATCHING));
const auto& batch_mode = config.find(ov::hint::allow_auto_batching.name());
if (batch_mode != config.end()) {
const auto disabled = batch_mode->second == CONFIG_VALUE(NO);
// virtual plugins like AUTO/MULTI will need the config
+6 -3
View File
@@ -584,13 +584,16 @@ void MultiDeviceInferencePlugin::CheckConfig(const std::map<std::string, std::st
} else if (kvp.first == ov::hint::model_priority.name()) {
try {
int priority = -1;
if (kvp.second == "LOW") {
if (kvp.second == "LOW" ||
kvp.second == CONFIG_VALUE(MODEL_PRIORITY_HIGH)) {
priority = static_cast<int>(ov::hint::Priority::HIGH) - static_cast<int>(ov::hint::Priority::LOW);
}
if (kvp.second == "MEDIUM") {
if (kvp.second == "MEDIUM" ||
kvp.second == CONFIG_VALUE(MODEL_PRIORITY_MED)) {
priority = static_cast<int>(ov::hint::Priority::HIGH) - static_cast<int>(ov::hint::Priority::MEDIUM);
}
if (kvp.second == "HIGH") {
if (kvp.second == "HIGH" ||
kvp.second == CONFIG_VALUE(MODEL_PRIORITY_HIGH)) {
priority = static_cast<int>(ov::hint::Priority::HIGH) - static_cast<int>(ov::hint::Priority::HIGH);
}
if (priority < 0) {
+18 -29
View File
@@ -8,6 +8,7 @@
#include <gna/gna_config.hpp>
#include "gna_plugin.hpp"
#include "gna_plugin_config.hpp"
#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp"
#include "ie_common.h"
#include <caseless.hpp>
#include <unordered_map>
@@ -55,21 +56,6 @@ inline std::istream& operator>>(std::istream& is, ov::element::Type& p) {
return is;
}
template <typename T>
T property_from_string(const std::string& string_value) {
std::stringstream ss(string_value);
T value;
ss >> value;
return value;
}
template <typename T>
std::string property_to_string(const T& property) {
std::stringstream ss;
ss << property;
return ss.str();
}
void Config::UpdateFromMap(const std::map<std::string, std::string>& config) {
for (auto&& item : config) {
auto key = item.first;
@@ -123,7 +109,7 @@ void Config::UpdateFromMap(const std::map<std::string, std::string>& config) {
};
if (key == ov::intel_gna::scale_factors_per_input) {
inputScaleFactorsPerInput = property_from_string<std::map<std::string, float>>(value);
inputScaleFactorsPerInput = InferenceEngine::util::string_to_property(value, ov::intel_gna::scale_factors_per_input);
for (auto&& sf : inputScaleFactorsPerInput) {
check_scale_factor(sf.second);
}
@@ -175,8 +161,8 @@ OPENVINO_SUPPRESS_DEPRECATED_START
swExactMode = procType->second.second;
}
OPENVINO_SUPPRESS_DEPRECATED_END
} else if (key == ov::intel_gna::execution_target || key == ov::intel_gna::compile_target) {
auto target = property_from_string<ov::intel_gna::HWGeneration>(value);
} else if (key == ov::intel_gna::execution_target || key == ov::intel_gna::compile_target) {
auto target = InferenceEngine::util::string_to_property(value, ov::intel_gna::execution_target);
std::string target_str = "";
if (ov::intel_gna::HWGeneration::GNA_2_0 == target) {
target_str = GNAConfigParams::GNA_TARGET_2_0;
@@ -209,9 +195,10 @@ OPENVINO_SUPPRESS_DEPRECATED_END
THROW_GNA_EXCEPTION << "EXCLUSIVE_ASYNC_REQUESTS should be YES/NO, but not" << value;
}
} else if (key == ov::hint::performance_mode) {
performance_mode = property_from_string<ov::hint::PerformanceMode>(value);
performance_mode = InferenceEngine::util::string_to_property(value, ov::hint::performance_mode);
} else if (key == ov::hint::inference_precision) {
inference_precision = property_from_string<ov::element::Type>(value);
std::stringstream ss(value);
ss >> inference_precision;
if ((inference_precision != ov::element::i8) && (inference_precision != ov::element::i16)) {
THROW_GNA_EXCEPTION << "Unsupported precision of GNA hardware, should be i16 or i8, but was: "
<< value;
@@ -227,7 +214,7 @@ OPENVINO_SUPPRESS_DEPRECATED_END
}
gnaPrecision = precision;
} else if (key == ov::intel_gna::pwl_design_algorithm) {
gnaFlags.pwl_design_algorithm = property_from_string<ov::intel_gna::PWLDesignAlgorithm>(value);
gnaFlags.pwl_design_algorithm = InferenceEngine::util::string_to_property(value, ov::intel_gna::pwl_design_algorithm);
gnaFlags.uniformPwlDesign = (gnaFlags.pwl_design_algorithm == ov::intel_gna::PWLDesignAlgorithm::UNIFORM_DISTRIBUTION) ? true : false;
OPENVINO_SUPPRESS_DEPRECATED_START
} else if (key == GNA_CONFIG_KEY(PWL_UNIFORM_DESIGN)) {
@@ -302,7 +289,7 @@ OPENVINO_SUPPRESS_DEPRECATED_START
OPENVINO_SUPPRESS_DEPRECATED_END
} else if (key == CONFIG_KEY(LOG_LEVEL) || key == ov::log::level) {
if (value == PluginConfigParams::LOG_WARNING || value == PluginConfigParams::LOG_NONE) {
gnaFlags.log_level = property_from_string<ov::log::Level>(value);
gnaFlags.log_level = InferenceEngine::util::string_to_property(value, ov::log::level);
} else {
log << "Currently only LOG_LEVEL = LOG_WARNING and LOG_NONE are supported, not " << value;
THROW_GNA_EXCEPTION << "Currently only LOG_LEVEL = LOG_WARNING and LOG_NONE are supported, not " << value;
@@ -330,7 +317,8 @@ void Config::AdjustKeyMapValues() {
keyConfigMap.clear();
if (!inputScaleFactorsPerInput.empty()) {
keyConfigMap[ov::intel_gna::scale_factors_per_input.name()] = property_to_string(inputScaleFactorsPerInput);
keyConfigMap[ov::intel_gna::scale_factors_per_input.name()] =
InferenceEngine::util::property_to_string(inputScaleFactorsPerInput);
} else {
if (inputScaleFactors.empty()) {
inputScaleFactors.push_back(1.0);
@@ -347,12 +335,12 @@ void Config::AdjustKeyMapValues() {
IE_SUPPRESS_DEPRECATED_END
std::string device_mode;
if (gnaFlags.sw_fp32) {
device_mode = property_to_string(ov::intel_gna::ExecutionMode::SW_FP32);
device_mode = InferenceEngine::util::property_to_string(ov::intel_gna::ExecutionMode::SW_FP32);
} else {
for (auto&& value : supported_values) {
if (value.second.first == pluginGna2AccMode &&
value.second.second == swExactMode) {
device_mode = property_to_string(value.first);
device_mode = InferenceEngine::util::property_to_string(value.first);
break;
}
}
@@ -365,15 +353,16 @@ void Config::AdjustKeyMapValues() {
gnaFlags.compact_mode ? PluginConfigParams::YES : PluginConfigParams::NO;
keyConfigMap[CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS)] =
gnaFlags.exclusive_async_requests ? PluginConfigParams::YES: PluginConfigParams::NO;
keyConfigMap[ov::hint::performance_mode.name()] = property_to_string(performance_mode);
keyConfigMap[ov::hint::performance_mode.name()] = InferenceEngine::util::property_to_string(performance_mode);
if (inference_precision != ov::element::undefined) {
keyConfigMap[ov::hint::inference_precision.name()] = property_to_string(inference_precision);
keyConfigMap[ov::hint::inference_precision.name()] = InferenceEngine::util::property_to_string(inference_precision);
} else {
keyConfigMap[GNA_CONFIG_KEY(PRECISION)] = gnaPrecision.name();
}
OPENVINO_SUPPRESS_DEPRECATED_START
if (gnaFlags.pwl_design_algorithm != ov::intel_gna::PWLDesignAlgorithm::UNDEFINED) {
keyConfigMap[ov::intel_gna::pwl_design_algorithm.name()] = property_to_string(gnaFlags.pwl_design_algorithm);
keyConfigMap[ov::intel_gna::pwl_design_algorithm.name()] =
InferenceEngine::util::property_to_string(gnaFlags.pwl_design_algorithm);
} else {
keyConfigMap[GNA_CONFIG_KEY(PWL_UNIFORM_DESIGN)] =
gnaFlags.uniformPwlDesign ? PluginConfigParams::YES: PluginConfigParams::NO;
@@ -386,7 +375,7 @@ OPENVINO_SUPPRESS_DEPRECATED_START
OPENVINO_SUPPRESS_DEPRECATED_END
keyConfigMap[ov::enable_profiling.name()] =
gnaFlags.performance_counting ? PluginConfigParams::YES: PluginConfigParams::NO;
keyConfigMap[ov::log::level.name()] = property_to_string(gnaFlags.log_level);
keyConfigMap[ov::log::level.name()] = InferenceEngine::util::property_to_string(gnaFlags.log_level);
}
Parameter Config::GetParameter(const std::string& name) const {
@@ -55,6 +55,7 @@ struct Config {
void UpdateFromMap(const std::map<std::string, std::string>& configMap);
void adjustKeyMapValues();
static bool isNewApiProperty(std::string property);
static std::string ConvertPropertyToLegacy(const std::string& key, const std::string& value);
std::string device_id;
uint16_t throughput_streams;
@@ -13,6 +13,7 @@
#include <description_buffer.hpp>
#include <threading/ie_executor_manager.hpp>
#include "threading/ie_cpu_streams_executor.hpp"
#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp"
#include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp"
#include "ie_icore.hpp"
@@ -137,9 +138,45 @@ std::shared_ptr<ngraph::Function> CompiledModel::GetExecGraphInfo() {
}
InferenceEngine::Parameter CompiledModel::GetConfig(const std::string &name) const {
const bool is_new_api = _plugin->GetCore()->isNewAPI();
auto it = m_config.key_config_map.find(name);
if (it != m_config.key_config_map.end()) {
return it->second;
std::string val = it->second;
if (is_new_api) {
if (name == ov::enable_profiling) {
return val == PluginConfigParams::YES ? true : false;
} else if (name == ov::hint::model_priority) {
return InferenceEngine::util::string_to_property(val, ov::hint::model_priority);
} else if (name == ov::intel_gpu::hint::host_task_priority) {
return InferenceEngine::util::string_to_property(val, ov::intel_gpu::hint::host_task_priority);
} else if (name == ov::intel_gpu::hint::queue_priority) {
return InferenceEngine::util::string_to_property(val, ov::intel_gpu::hint::queue_priority);
} else if (name == ov::intel_gpu::hint::queue_throttle) {
return InferenceEngine::util::string_to_property(val, ov::intel_gpu::hint::queue_throttle);
} else if (name == ov::intel_gpu::enable_loop_unrolling) {
return val == PluginConfigParams::YES ? true : false;
} else if (name == ov::cache_dir) {
return InferenceEngine::util::string_to_property(val, ov::cache_dir);
} else if (name == ov::hint::performance_mode) {
return InferenceEngine::util::string_to_property(val, ov::hint::performance_mode);
} else if (name == ov::compilation_num_threads) {
return InferenceEngine::util::string_to_property(val, ov::compilation_num_threads);
} else if (name == ov::streams::num) {
return InferenceEngine::util::string_to_property(val, ov::streams::num);
} else if (name == ov::hint::num_requests) {
return InferenceEngine::util::string_to_property(val, ov::hint::num_requests);
} else if (name == ov::device::id) {
return InferenceEngine::util::string_to_property(val, ov::device::id);
} else {
return val;
}
} else {
if (name == PluginConfigParams::KEY_MODEL_PRIORITY ||
name == GPUConfigParams::KEY_GPU_HOST_TASK_PRIORITY)
return Config::ConvertPropertyToLegacy(name, val);
else
return val;
}
} else {
IE_THROW() << "Unsupported ExecutableNetwork config key: " << name;
}
@@ -154,15 +191,18 @@ InferenceEngine::Parameter CompiledModel::GetMetric(const std::string &name) con
ov::PropertyName{ov::optimal_number_of_infer_requests.name(), PropertyMutability::RO},
// Configs
PropertyName{ov::enable_profiling.name(), PropertyMutability::RW},
PropertyName{ov::hint::model_priority.name(), PropertyMutability::RW},
PropertyName{ov::intel_gpu::hint::host_task_priority.name(), PropertyMutability::RW},
PropertyName{ov::intel_gpu::hint::queue_priority.name(), PropertyMutability::RW},
PropertyName{ov::intel_gpu::hint::queue_throttle.name(), PropertyMutability::RW},
PropertyName{ov::intel_gpu::enable_loop_unrolling.name(), PropertyMutability::RW},
PropertyName{ov::cache_dir.name(), PropertyMutability::RW},
PropertyName{ov::hint::performance_mode.name(), PropertyMutability::RW},
PropertyName{ov::compilation_num_threads.name(), PropertyMutability::RW}
ov::PropertyName{ov::enable_profiling.name(), PropertyMutability::RO},
ov::PropertyName{ov::hint::model_priority.name(), PropertyMutability::RO},
ov::PropertyName{ov::intel_gpu::hint::host_task_priority.name(), PropertyMutability::RO},
ov::PropertyName{ov::intel_gpu::hint::queue_priority.name(), PropertyMutability::RO},
ov::PropertyName{ov::intel_gpu::hint::queue_throttle.name(), PropertyMutability::RO},
ov::PropertyName{ov::intel_gpu::enable_loop_unrolling.name(), PropertyMutability::RO},
ov::PropertyName{ov::cache_dir.name(), PropertyMutability::RO},
ov::PropertyName{ov::hint::performance_mode.name(), PropertyMutability::RO},
ov::PropertyName{ov::compilation_num_threads.name(), PropertyMutability::RO},
ov::PropertyName{ov::streams::num.name(), PropertyMutability::RO},
ov::PropertyName{ov::hint::num_requests.name(), PropertyMutability::RO},
ov::PropertyName{ov::device::id.name(), PropertyMutability::RO}
};
} else if (name == ov::model_name) {
IE_ASSERT(!m_graphs.empty());
@@ -126,44 +126,27 @@ void Config::UpdateFromMap(const std::map<std::string, std::string>& configMap)
queuePriority = cldnn::priority_mode_types::low;
} else if (key.compare(PluginConfigParams::KEY_MODEL_PRIORITY) == 0 ||
key == ov::hint::model_priority) {
if (key == ov::hint::model_priority) {
std::stringstream ss(val);
ov::hint::Priority priority;
ss >> priority;
switch (priority) {
case ov::hint::Priority::LOW:
queuePriority = cldnn::priority_mode_types::low;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::LITTLE;
break;
case ov::hint::Priority::MEDIUM:
queuePriority = cldnn::priority_mode_types::med;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::ANY;
break;
case ov::hint::Priority::HIGH:
queuePriority = cldnn::priority_mode_types::high;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::BIG;
break;
}
if (val.compare(PluginConfigParams::MODEL_PRIORITY_HIGH) == 0 ||
val.compare(InferenceEngine::util::property_to_string(ov::hint::Priority::HIGH)) == 0) {
queuePriority = cldnn::priority_mode_types::high;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::BIG;
} else if (val.compare(PluginConfigParams::MODEL_PRIORITY_MED) == 0 ||
val.compare(InferenceEngine::util::property_to_string(ov::hint::Priority::MEDIUM)) == 0) {
queuePriority = cldnn::priority_mode_types::med;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::ANY;
} else if (val.compare(PluginConfigParams::MODEL_PRIORITY_LOW) == 0 ||
val.compare(InferenceEngine::util::property_to_string(ov::hint::Priority::LOW)) == 0) {
queuePriority = cldnn::priority_mode_types::low;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::LITTLE;
} else {
if (val.find(PluginConfigParams::MODEL_PRIORITY_HIGH) != std::string::npos) {
queuePriority = cldnn::priority_mode_types::high;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::BIG;
} else if (val.find(PluginConfigParams::MODEL_PRIORITY_LOW) != std::string::npos) {
queuePriority = cldnn::priority_mode_types::low;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::LITTLE;
} else if (val.find(PluginConfigParams::MODEL_PRIORITY_MED) != std::string::npos) {
queuePriority = cldnn::priority_mode_types::med;
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::ANY;
} else {
IE_THROW() << "Not found appropriate value for config key " << PluginConfigParams::KEY_MODEL_PRIORITY << ".\n";
}
IE_THROW() << "Not found appropriate value for config key " << PluginConfigParams::KEY_MODEL_PRIORITY << ".\n";
}
if (getAvailableCoresTypes().size() > 1) {
if (task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::BIG
|| task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::LITTLE) {
task_exec_config._streams = std::min(task_exec_config._streams,
getNumberOfCores(task_exec_config._threadPreferredCoreType));
}
getNumberOfCores(task_exec_config._threadPreferredCoreType));
}
} else {
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::ANY;
task_exec_config._streams = std::min(task_exec_config._streams,
@@ -259,7 +242,8 @@ void Config::UpdateFromMap(const std::map<std::string, std::string>& configMap)
}
} else if (key.compare(PluginConfigParams::KEY_GPU_THROUGHPUT_STREAMS) == 0 ||
key == ov::streams::num) {
if (val.compare(PluginConfigParams::GPU_THROUGHPUT_AUTO) == 0) {
if (val.compare(PluginConfigParams::GPU_THROUGHPUT_AUTO) == 0 ||
val.compare(std::to_string(ov::streams::AUTO)) == 0) {
throughput_streams = GetDefaultNStreamsForThroughputMode();
} else {
int val_i;
@@ -333,27 +317,20 @@ void Config::UpdateFromMap(const std::map<std::string, std::string>& configMap)
} else {
IE_THROW(ParameterMismatch) << "Unsupported KEY_GPU_ENABLE_LOOP_UNROLLING flag value: " << val;
}
} else if (key == ov::intel_gpu::hint::host_task_priority) {
std::stringstream ss(val);
ov::hint::Priority priority;
ss >> priority;
if (priority == ov::hint::Priority::HIGH)
} else if (key.compare(GPUConfigParams::KEY_GPU_HOST_TASK_PRIORITY) == 0 ||
key == ov::intel_gpu::hint::host_task_priority) {
if (val.compare(GPUConfigParams::GPU_HOST_TASK_PRIORITY_HIGH) == 0 ||
val.compare(InferenceEngine::util::property_to_string(ov::hint::Priority::HIGH)) == 0) {
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::BIG;
else if (priority == ov::hint::Priority::LOW)
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::LITTLE;
else if (priority == ov::hint::Priority::MEDIUM)
} else if (val.compare(GPUConfigParams::GPU_HOST_TASK_PRIORITY_MEDIUM) == 0 ||
val.compare(InferenceEngine::util::property_to_string(ov::hint::Priority::MEDIUM)) == 0) {
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::ANY;
else
IE_THROW(NotFound) << "Unsupported host task priority by plugin: " << val;
} else if (key.compare(GPUConfigParams::KEY_GPU_HOST_TASK_PRIORITY) == 0) {
if (val.compare(GPUConfigParams::GPU_HOST_TASK_PRIORITY_HIGH) == 0)
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::BIG;
else if (val.compare(GPUConfigParams::GPU_HOST_TASK_PRIORITY_MEDIUM) == 0)
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::ANY;
else if (val.compare(GPUConfigParams::GPU_HOST_TASK_PRIORITY_LOW) == 0)
} else if (val.compare(GPUConfigParams::GPU_HOST_TASK_PRIORITY_LOW) == 0 ||
val.compare(InferenceEngine::util::property_to_string(ov::hint::Priority::LOW)) == 0) {
task_exec_config._threadPreferredCoreType = IStreamsExecutor::Config::LITTLE;
else
} else {
IE_THROW(NotFound) << "Unsupported host task priority by plugin: " << val;
}
} else {
IE_THROW(NotFound) << "Unsupported property key by plugin: " << key;
}
@@ -406,19 +383,14 @@ void Config::adjustKeyMapValues() {
key_config_map[CLDNNConfigParams::KEY_CLDNN_ENABLE_FP16_FOR_QUANTIZED_MODELS] = PluginConfigParams::NO;
{
std::stringstream s;
if (queuePriority == cldnn::priority_mode_types::high && task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::BIG) {
s << ov::hint::Priority::HIGH;
key_config_map[ov::hint::model_priority.name()] = s.str();
key_config_map[PluginConfigParams::KEY_MODEL_PRIORITY] = PluginConfigParams::MODEL_PRIORITY_HIGH;
} else if (queuePriority == cldnn::priority_mode_types::low && task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::LITTLE) {
s << ov::hint::Priority::LOW;
key_config_map[ov::hint::model_priority.name()] = s.str();
key_config_map[PluginConfigParams::KEY_MODEL_PRIORITY] = PluginConfigParams::MODEL_PRIORITY_LOW;
if (queuePriority == cldnn::priority_mode_types::high &&
(task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::BIG || getAvailableCoresTypes().size() == 1)) {
key_config_map[ov::hint::model_priority.name()] = InferenceEngine::util::property_to_string(ov::hint::Priority::HIGH);
} else if (queuePriority == cldnn::priority_mode_types::low &&
(task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::LITTLE || getAvailableCoresTypes().size() == 1)) {
key_config_map[ov::hint::model_priority.name()] = InferenceEngine::util::property_to_string(ov::hint::Priority::LOW);
} else if (queuePriority == cldnn::priority_mode_types::med && task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::ANY) {
s << ov::hint::Priority::MEDIUM;
key_config_map[ov::hint::model_priority.name()] = s.str();
key_config_map[PluginConfigParams::KEY_MODEL_PRIORITY] = PluginConfigParams::MODEL_PRIORITY_MED;
key_config_map[ov::hint::model_priority.name()] = InferenceEngine::util::property_to_string(ov::hint::Priority::MEDIUM);
}
}
{
@@ -433,14 +405,14 @@ void Config::adjustKeyMapValues() {
key_config_map[GPUConfigParams::KEY_GPU_PLUGIN_PRIORITY] = qp;
}
{
std::stringstream ss;
std::string priority;
if (queuePriority == cldnn::priority_mode_types::high)
ss << ov::hint::Priority::HIGH;
priority = InferenceEngine::util::property_to_string(ov::hint::Priority::HIGH);
else if (queuePriority == cldnn::priority_mode_types::low)
ss << ov::hint::Priority::LOW;
priority = InferenceEngine::util::property_to_string(ov::hint::Priority::LOW);
else
ss << ov::hint::Priority::MEDIUM;
key_config_map[ov::intel_gpu::hint::queue_priority.name()] = ss.str();
priority = InferenceEngine::util::property_to_string(ov::hint::Priority::MEDIUM);
key_config_map[ov::intel_gpu::hint::queue_priority.name()] = priority;
}
{
std::string qt = "0";
@@ -454,30 +426,24 @@ void Config::adjustKeyMapValues() {
key_config_map[GPUConfigParams::KEY_GPU_PLUGIN_THROTTLE] = qt;
}
{
std::stringstream ss;
std::string throttleLevel;
if (queueThrottle == cldnn::throttle_mode_types::high)
ss << ov::intel_gpu::hint::ThrottleLevel::HIGH;
throttleLevel = InferenceEngine::util::property_to_string(ov::intel_gpu::hint::ThrottleLevel::HIGH);
else if (queueThrottle == cldnn::throttle_mode_types::low)
ss << ov::intel_gpu::hint::ThrottleLevel::LOW;
throttleLevel = InferenceEngine::util::property_to_string(ov::intel_gpu::hint::ThrottleLevel::LOW);
else
ss << ov::intel_gpu::hint::ThrottleLevel::MEDIUM;
key_config_map[ov::intel_gpu::hint::queue_throttle.name()] = ss.str();
throttleLevel = InferenceEngine::util::property_to_string(ov::intel_gpu::hint::ThrottleLevel::MEDIUM);
key_config_map[ov::intel_gpu::hint::queue_throttle.name()] = throttleLevel;
}
{
std::stringstream s;
if (task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::LITTLE) {
s << ov::hint::Priority::LOW;
key_config_map[ov::intel_gpu::hint::host_task_priority.name()] = s.str();
key_config_map[GPUConfigParams::KEY_GPU_HOST_TASK_PRIORITY] = GPUConfigParams::GPU_HOST_TASK_PRIORITY_LOW;
} else if (task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::BIG) {
s << ov::hint::Priority::HIGH;
key_config_map[ov::intel_gpu::hint::host_task_priority.name()] = s.str();
key_config_map[GPUConfigParams::KEY_GPU_HOST_TASK_PRIORITY] = GPUConfigParams::GPU_HOST_TASK_PRIORITY_HIGH;
} else {
s << ov::hint::Priority::MEDIUM;
key_config_map[ov::intel_gpu::hint::host_task_priority.name()] = s.str();
key_config_map[GPUConfigParams::KEY_GPU_HOST_TASK_PRIORITY] = GPUConfigParams::GPU_HOST_TASK_PRIORITY_MEDIUM;
}
std::string hostTaskPriority;
if (task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::LITTLE)
hostTaskPriority = InferenceEngine::util::property_to_string(ov::hint::Priority::LOW);
else if (task_exec_config._threadPreferredCoreType == IStreamsExecutor::Config::BIG)
hostTaskPriority = InferenceEngine::util::property_to_string(ov::hint::Priority::HIGH);
else
hostTaskPriority = InferenceEngine::util::property_to_string(ov::hint::Priority::MEDIUM);
key_config_map[ov::intel_gpu::hint::host_task_priority.name()] = hostTaskPriority;
}
{
std::string tm = PluginConfigParams::TUNING_DISABLED;
@@ -516,8 +482,8 @@ void Config::adjustKeyMapValues() {
key_config_map[ov::intel_gpu::enable_loop_unrolling.name()] = PluginConfigParams::NO;
}
key_config_map[PluginConfigParams::KEY_PERFORMANCE_HINT]= perfHintsConfig.ovPerfHint;
key_config_map[ov::hint::performance_mode.name()]= perfHintsConfig.ovPerfHint;
key_config_map[PluginConfigParams::KEY_PERFORMANCE_HINT] = perfHintsConfig.ovPerfHint;
key_config_map[ov::hint::performance_mode.name()] = perfHintsConfig.ovPerfHint;
key_config_map[PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS] =
std::to_string(perfHintsConfig.ovPerfHintNumRequests);
@@ -525,15 +491,35 @@ void Config::adjustKeyMapValues() {
bool Config::isNewApiProperty(std::string property) {
static const std::set<std::string> new_api_keys {
ov::hint::model_priority.name(),
ov::intel_gpu::hint::host_task_priority.name(),
ov::intel_gpu::hint::queue_priority.name(),
ov::intel_gpu::hint::queue_throttle.name(),
ov::compilation_num_threads.name(),
ov::streams::num.name(),
};
return new_api_keys.find(property) != new_api_keys.end();
}
std::string Config::ConvertPropertyToLegacy(const std::string& key, const std::string& value) {
if (key == PluginConfigParams::KEY_MODEL_PRIORITY) {
auto priority = InferenceEngine::util::string_to_property(value, ov::hint::model_priority);
if (priority == ov::hint::Priority::HIGH)
return PluginConfigParams::MODEL_PRIORITY_HIGH;
else if (priority == ov::hint::Priority::MEDIUM)
return PluginConfigParams::MODEL_PRIORITY_MED;
else if (priority == ov::hint::Priority::LOW)
return PluginConfigParams::MODEL_PRIORITY_LOW;
} else if (key == GPUConfigParams::KEY_GPU_HOST_TASK_PRIORITY) {
auto priority = InferenceEngine::util::string_to_property(value, ov::intel_gpu::hint::host_task_priority);
if (priority == ov::hint::Priority::HIGH)
return GPUConfigParams::GPU_HOST_TASK_PRIORITY_HIGH;
else if (priority == ov::hint::Priority::MEDIUM)
return GPUConfigParams::GPU_HOST_TASK_PRIORITY_MEDIUM;
else if (priority == ov::hint::Priority::LOW)
return GPUConfigParams::GPU_HOST_TASK_PRIORITY_LOW;
}
IE_THROW() << "Unsupported value for legacy key : " << key;
}
void Configs::CreateConfig(std::string device_id) {
if (configs.find(device_id) == configs.end()) {
configs.emplace(device_id, Config(device_id));
+72 -22
View File
@@ -213,12 +213,15 @@ std::map<std::string, std::string> Plugin::ConvertPerfHintsToConfig(
? PerfHintsConfig::CheckPerformanceHintValue(mode->second)
: plugin_config.perfHintsConfig.ovPerfHint;
//checking streams (to avoid overriding what user might explicitly set in the incoming config or previously via SetConfig)
const auto streams = config.find(PluginConfigParams::KEY_GPU_THROUGHPUT_STREAMS);
if (streams == config.end() && !streamsSet) {
const auto streams = config.find(PluginConfigParams::KEY_GPU_THROUGHPUT_STREAMS) == config.end() &&
config.find(ov::streams::num.name()) == config.end();
if (streams && !streamsSet) {
if (mode_name == CONFIG_VALUE(LATENCY)) {
config[PluginConfigParams::KEY_GPU_THROUGHPUT_STREAMS] = std::to_string(1);
config[ov::streams::num.name()] = std::to_string(1);
} else if (mode_name == CONFIG_VALUE(THROUGHPUT)) {
config[PluginConfigParams::KEY_GPU_THROUGHPUT_STREAMS] = CONFIG_VALUE(GPU_THROUGHPUT_AUTO);
config[ov::streams::num.name()] = std::to_string(ov::streams::AUTO);
//disabling the throttling temporarily to set the validation (that is switching to the hints) perf baseline
//checking throttling (to avoid overriding what user might explicitly set in the incoming config or previously via SetConfig)
// const auto bInConfig = config.find(GPUConfigParams::KEY_GPU_PLUGIN_THROTTLE) != config.end() ||
@@ -533,12 +536,48 @@ Parameter Plugin::GetConfig(const std::string& name, const std::map<std::string,
}
Config config = _impl->m_configs.GetConfig(device_id);
const bool is_new_api = GetCore()->isNewAPI();
if (config.key_config_map.find(name) != config.key_config_map.end()) {
result = config.key_config_map.find(name)->second;
std::string val = config.key_config_map.find(name)->second;
if (is_new_api) {
if (name == ov::enable_profiling) {
return val == PluginConfigParams::YES ? true : false;
} else if (name == ov::hint::model_priority) {
return InferenceEngine::util::string_to_property(val, ov::hint::model_priority);
} else if (name == ov::intel_gpu::hint::host_task_priority) {
return InferenceEngine::util::string_to_property(val, ov::intel_gpu::hint::host_task_priority);
} else if (name == ov::intel_gpu::hint::queue_priority) {
return InferenceEngine::util::string_to_property(val, ov::intel_gpu::hint::queue_priority);
} else if (name == ov::intel_gpu::hint::queue_throttle) {
return InferenceEngine::util::string_to_property(val, ov::intel_gpu::hint::queue_throttle);
} else if (name == ov::intel_gpu::enable_loop_unrolling) {
return val == PluginConfigParams::YES ? true : false;
} else if (name == ov::cache_dir) {
return InferenceEngine::util::string_to_property(val, ov::cache_dir);
} else if (name == ov::hint::performance_mode) {
return InferenceEngine::util::string_to_property(val, ov::hint::performance_mode);
} else if (name == ov::compilation_num_threads) {
return InferenceEngine::util::string_to_property(val, ov::compilation_num_threads);
} else if (name == ov::streams::num) {
return InferenceEngine::util::string_to_property(val, ov::streams::num);
} else if (name == ov::hint::num_requests) {
auto temp = InferenceEngine::util::string_to_property(val, ov::hint::num_requests);;
return temp;
} else if (name == ov::device::id) {
return InferenceEngine::util::string_to_property(val, ov::device::id);
} else {
return val;
}
} else {
if (name == PluginConfigParams::KEY_MODEL_PRIORITY ||
name == GPUConfigParams::KEY_GPU_HOST_TASK_PRIORITY)
return Config::ConvertPropertyToLegacy(name, val);
else
return val;
}
} else {
IE_THROW() << "Unsupported config key : " << name;
}
return result;
}
auto StringRightTrim = [](std::string string, std::string substring, bool case_sensitive = true) {
@@ -639,15 +678,18 @@ Parameter Plugin::GetMetric(const std::string& name, const std::map<std::string,
ov::PropertyName{ov::intel_gpu::memory_statistics.name(), PropertyMutability::RO},
// Configs
PropertyName{ov::enable_profiling.name(), PropertyMutability::RW},
PropertyName{ov::hint::model_priority.name(), PropertyMutability::RW},
PropertyName{ov::intel_gpu::hint::host_task_priority.name(), PropertyMutability::RW},
PropertyName{ov::intel_gpu::hint::queue_priority.name(), PropertyMutability::RW},
PropertyName{ov::intel_gpu::hint::queue_throttle.name(), PropertyMutability::RW},
PropertyName{ov::intel_gpu::enable_loop_unrolling.name(), PropertyMutability::RW},
PropertyName{ov::cache_dir.name(), PropertyMutability::RW},
PropertyName{ov::hint::performance_mode.name(), PropertyMutability::RW},
PropertyName{ov::compilation_num_threads.name(), PropertyMutability::RW}
ov::PropertyName{ov::enable_profiling.name(), PropertyMutability::RW},
ov::PropertyName{ov::hint::model_priority.name(), PropertyMutability::RW},
ov::PropertyName{ov::intel_gpu::hint::host_task_priority.name(), PropertyMutability::RW},
ov::PropertyName{ov::intel_gpu::hint::queue_priority.name(), PropertyMutability::RW},
ov::PropertyName{ov::intel_gpu::hint::queue_throttle.name(), PropertyMutability::RW},
ov::PropertyName{ov::intel_gpu::enable_loop_unrolling.name(), PropertyMutability::RW},
ov::PropertyName{ov::cache_dir.name(), PropertyMutability::RW},
ov::PropertyName{ov::hint::performance_mode.name(), PropertyMutability::RW},
ov::PropertyName{ov::compilation_num_threads.name(), PropertyMutability::RW},
ov::PropertyName{ov::streams::num.name(), PropertyMutability::RW},
ov::PropertyName{ov::hint::num_requests.name(), PropertyMutability::RW},
ov::PropertyName{ov::device::id.name(), PropertyMutability::RW},
};
} else if (name == METRIC_KEY(SUPPORTED_METRICS)) {
std::vector<std::string> metrics;
@@ -855,9 +897,16 @@ Parameter Plugin::GetMetric(const std::string& name, const std::map<std::string,
}
return decltype(ov::max_batch_size)::value_type {static_cast<uint32_t>(max_batch_size)};
}
auto it_streams = options.find("GPU_THROUGHPUT_STREAMS");
auto it_streams = options.find("GPU_THROUGHPUT_STREAMS") != options.end() ? options.find("GPU_THROUGHPUT_STREAMS") :
options.find(ov::streams::num.name()) != options.end() ? options.find(ov::streams::num.name()) :
options.end();
if (it_streams != options.end()) {
if (it_streams->second.is<uint32_t>()) {
if (it_streams->second.is<int32_t>()) {
n_streams = it_streams->second.as<int32_t>();
if (n_streams == ov::streams::AUTO)
n_streams = config.GetDefaultNStreamsForThroughputMode();
} else if (it_streams->second.is<uint32_t>()) {
n_streams = it_streams->second.as<uint32_t>();
} else if (it_streams->second.is<std::string>()) {
std::string n_streams_str = it_streams->second.as<std::string>();
@@ -869,18 +918,19 @@ Parameter Plugin::GetMetric(const std::string& name, const std::map<std::string,
IE_THROW() << "[GPU_MAX_BATCH_SIZE] bad casting: GPU_THROUGHPUT_STREAMS should be either of uint32_t type or \"GPU_THROUGHPUT_AUTO\"";
}
}
GPU_DEBUG_IF(debug_config->verbose >= 2) {
GPU_DEBUG_COUT << "[GPU_MAX_BATCH_SIZE] n_streams : " << n_streams << std::endl;
}
if (options.find(ov::intel_gpu::hint::available_device_mem.name()) != options.end()) {
try {
available_device_mem = std::min(static_cast<int64_t>(available_device_mem),
options.find(ov::intel_gpu::hint::available_device_mem.name())->second.as<int64_t>());
auto available_device_mem_it = options.find(ov::intel_gpu::hint::available_device_mem.name());
if (available_device_mem_it != options.end()) {
if (available_device_mem_it->second.is<int64_t>()) {
available_device_mem = std::min(static_cast<int64_t>(available_device_mem), available_device_mem_it->second.as<int64_t>());
GPU_DEBUG_IF(debug_config->verbose >= 2) {
GPU_DEBUG_COUT << "[GPU_MAX_BATCH_SIZE] available memory is reset by user " << available_device_mem << std::endl;
}
} catch (...) {
} else {
IE_THROW() << "[GPU_MAX_BATCH_SIZE] bad casting: ov::intel_gpu::hint::available_device_mem should be int64_t type";
}
if (available_device_mem < 0) {
@@ -890,9 +940,9 @@ Parameter Plugin::GetMetric(const std::string& name, const std::map<std::string,
std::shared_ptr<ngraph::Function> model;
auto model_param = options.find(ov::hint::model.name())->second;
try {
if (model_param.is<std::shared_ptr<ngraph::Function>>()) {
model = model_param.as<std::shared_ptr<ngraph::Function>>();
} catch (...) {
} else {
IE_THROW() << "[GPU_MAX_BATCH_SIZE] ov::hint::model should be std::shared_ptr<ov::Model> type";
}
@@ -51,6 +51,10 @@ INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
OVClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
::testing::Values("GPU"));
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
OVClassGetMetricTest_MAX_BATCH_SIZE,
::testing::Values("GPU"));
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest, OVClassGetMetricTest_DEVICE_GOPS, ::testing::Values("GPU"));
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest, OVClassGetMetricTest_DEVICE_TYPE, ::testing::Values("GPU"));
@@ -275,15 +279,94 @@ TEST_P(OVClassGetPropertyTest_GPU, GetMetricMemoryStatisticsAndPrintNoThrow) {
OV_ASSERT_PROPERTY_SUPPORTED(ov::intel_gpu::memory_statistics);
}
TEST_P(OVClassGetPropertyTest_GPU, GetMetricMaxBatchSizeAndPrintNoThrow) {
TEST_P(OVClassGetPropertyTest_GPU, GetAndSetPerformanceModeNoThrow) {
ov::Core ie;
uint32_t property;
ASSERT_NO_THROW(property = ie.get_property(deviceName, ov::max_batch_size));
ov::hint::PerformanceMode defaultMode;
ASSERT_NO_THROW(defaultMode = ie.get_property(deviceName, ov::hint::performance_mode));
std::cout << "GPU_MAX_BATCH_SIZE: " << property << std::endl;
std::cout << "Default PERFORMANCE_HINT: \"" << defaultMode << "\"" << std::endl;
OV_ASSERT_PROPERTY_SUPPORTED(ov::max_batch_size);
ie.set_property(deviceName, ov::hint::performance_mode(ov::hint::PerformanceMode::UNDEFINED));
ASSERT_EQ(ov::hint::PerformanceMode::UNDEFINED, ie.get_property(deviceName, ov::hint::performance_mode));
ie.set_property(deviceName, ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
ASSERT_EQ(ov::hint::PerformanceMode::LATENCY, ie.get_property(deviceName, ov::hint::performance_mode));
ie.set_property(deviceName, ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT));
ASSERT_EQ(ov::hint::PerformanceMode::THROUGHPUT, ie.get_property(deviceName, ov::hint::performance_mode));
OV_ASSERT_PROPERTY_SUPPORTED(ov::hint::performance_mode);
}
TEST_P(OVClassGetPropertyTest_GPU, GetAndSetEnableProfilingNoThrow) {
ov::Core ie;
bool defaultValue = false;
ASSERT_NO_THROW(defaultValue = ie.get_property(deviceName, ov::enable_profiling));
std::cout << "Default PERF_COUNT: " << defaultValue << std::endl;
ie.set_property(deviceName, ov::enable_profiling(true));
ASSERT_EQ(true, ie.get_property(deviceName, ov::enable_profiling));
OV_ASSERT_PROPERTY_SUPPORTED(ov::enable_profiling);
}
TEST_P(OVClassGetPropertyTest_GPU, GetAndSetModelPriorityNoThrow) {
ov::Core ie;
ov::hint::Priority defaultValue;
ASSERT_NO_THROW(defaultValue = ie.get_property(deviceName, ov::hint::model_priority));
std::cout << "Default PERF_COUNT: " << defaultValue << std::endl;
ie.set_property(deviceName, ov::hint::model_priority(ov::hint::Priority::HIGH));
ASSERT_EQ(ov::hint::Priority::HIGH, ie.get_property(deviceName, ov::hint::model_priority));
ASSERT_EQ(ov::hint::Priority::HIGH, ie.get_property(deviceName, ov::intel_gpu::hint::queue_priority));
ie.set_property(deviceName, ov::hint::model_priority(ov::hint::Priority::LOW));
ASSERT_EQ(ov::hint::Priority::LOW, ie.get_property(deviceName, ov::hint::model_priority));
ASSERT_EQ(ov::hint::Priority::LOW, ie.get_property(deviceName, ov::intel_gpu::hint::queue_priority));
ie.set_property(deviceName, ov::hint::model_priority(ov::hint::Priority::MEDIUM));
ASSERT_EQ(ov::hint::Priority::MEDIUM, ie.get_property(deviceName, ov::hint::model_priority));
ASSERT_EQ(ov::hint::Priority::MEDIUM, ie.get_property(deviceName, ov::intel_gpu::hint::queue_priority));
OV_ASSERT_PROPERTY_SUPPORTED(ov::hint::model_priority);
}
TEST_P(OVClassGetPropertyTest_GPU, GetAndSetQueuePriorityNoThrow) {
ov::Core ie;
ov::hint::Priority defaultValue;
ASSERT_NO_THROW(defaultValue = ie.get_property(deviceName, ov::intel_gpu::hint::queue_priority));
std::cout << "Default GPU_QUEUE_PRIORITY: " << defaultValue << std::endl;
ie.set_property(deviceName, ov::intel_gpu::hint::queue_priority(ov::hint::Priority::HIGH));
ASSERT_EQ(ov::hint::Priority::HIGH, ie.get_property(deviceName, ov::intel_gpu::hint::queue_priority));
ie.set_property(deviceName, ov::intel_gpu::hint::queue_priority(ov::hint::Priority::LOW));
ASSERT_EQ(ov::hint::Priority::LOW, ie.get_property(deviceName, ov::intel_gpu::hint::queue_priority));
ie.set_property(deviceName, ov::intel_gpu::hint::queue_priority(ov::hint::Priority::MEDIUM));
ASSERT_EQ(ov::hint::Priority::MEDIUM, ie.get_property(deviceName, ov::intel_gpu::hint::queue_priority));
OV_ASSERT_PROPERTY_SUPPORTED(ov::intel_gpu::hint::queue_priority);
}
TEST_P(OVClassGetPropertyTest_GPU, GetAndSetThrottleLevelNoThrow) {
ov::Core ie;
ov::intel_gpu::hint::ThrottleLevel defaultValue;
ASSERT_NO_THROW(defaultValue = ie.get_property(deviceName, ov::intel_gpu::hint::queue_throttle));
std::cout << "Default GPU_QUEUE_THROTTLE: " << defaultValue << std::endl;
ie.set_property(deviceName, ov::intel_gpu::hint::queue_throttle(ov::intel_gpu::hint::ThrottleLevel::HIGH));
ASSERT_EQ(ov::intel_gpu::hint::ThrottleLevel::HIGH, ie.get_property(deviceName, ov::intel_gpu::hint::queue_throttle));
ie.set_property(deviceName, ov::intel_gpu::hint::queue_throttle(ov::intel_gpu::hint::ThrottleLevel::LOW));
ASSERT_EQ(ov::intel_gpu::hint::ThrottleLevel::LOW, ie.get_property(deviceName, ov::intel_gpu::hint::queue_throttle));
ie.set_property(deviceName, ov::intel_gpu::hint::queue_throttle(ov::intel_gpu::hint::ThrottleLevel::MEDIUM));
ASSERT_EQ(ov::intel_gpu::hint::ThrottleLevel::MEDIUM, ie.get_property(deviceName, ov::intel_gpu::hint::queue_throttle));
OV_ASSERT_PROPERTY_SUPPORTED(ov::intel_gpu::hint::queue_throttle);
}
TEST_P(OVClassGetPropertyTest_GPU, CanSetDefaultValueBackToPluginNewAPI) {
@@ -115,6 +115,71 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values("GPU")
);
using IEClassGetMetricTest_GPU_OPTIMAL_BATCH_SIZE = BehaviorTestsUtils::IEClassBaseTestP;
TEST_P(IEClassGetMetricTest_GPU_OPTIMAL_BATCH_SIZE, GetMetricAndPrintNoThrow) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
InferenceEngine::Core ie;
InferenceEngine::Parameter p;
std::map<std::string, InferenceEngine::Parameter> _options = {{"MODEL_PTR", simpleCnnNetwork.getFunction()}};
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, METRIC_KEY(OPTIMAL_BATCH_SIZE), _options).as<unsigned int>());
unsigned int t = p;
std::cout << "GPU device optimal batch size: " << t << std::endl;
ASSERT_METRIC_SUPPORTED_IE(METRIC_KEY(OPTIMAL_BATCH_SIZE));
}
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassExecutableNetworkGetMetricTest, IEClassGetMetricTest_GPU_OPTIMAL_BATCH_SIZE,
::testing::Values("GPU")
);
using IEClassGetMetricTest_GPU_MAX_BATCH_SIZE_DEFAULT = BehaviorTestsUtils::IEClassBaseTestP;
TEST_P(IEClassGetMetricTest_GPU_MAX_BATCH_SIZE_DEFAULT, GetMetricAndPrintNoThrow) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
InferenceEngine::Core ie;
InferenceEngine::Parameter p;
std::map<std::string, InferenceEngine::Parameter> _options = {{"MODEL_PTR", simpleCnnNetwork.getFunction()}};
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, METRIC_KEY(MAX_BATCH_SIZE), _options).as<uint32_t>());
uint32_t t = p;
std::cout << "GPU device max available batch size: " << t << std::endl;
ASSERT_METRIC_SUPPORTED_IE(METRIC_KEY(MAX_BATCH_SIZE));
}
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassExecutableNetworkGetMetricTest, IEClassGetMetricTest_GPU_MAX_BATCH_SIZE_DEFAULT,
::testing::Values("GPU")
);
using IEClassGetMetricTest_GPU_MAX_BATCH_SIZE_STREAM_DEVICE_MEM = BehaviorTestsUtils::IEClassBaseTestP;
TEST_P(IEClassGetMetricTest_GPU_MAX_BATCH_SIZE_STREAM_DEVICE_MEM, GetMetricAndPrintNoThrow) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
InferenceEngine::Core ie;
InferenceEngine::Parameter p;
uint32_t n_streams = 2;
int64_t available_device_mem_size = 1073741824;
std::map<std::string, InferenceEngine::Parameter> _options = {{"MODEL_PTR", simpleCnnNetwork.getFunction()}};
_options.insert(std::make_pair("GPU_THROUGHPUT_STREAMS", n_streams));
_options.insert(std::make_pair("AVAILABLE_DEVICE_MEM_SIZE", available_device_mem_size));
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, METRIC_KEY(MAX_BATCH_SIZE), _options).as<uint32_t>());
uint32_t t = p;
std::cout << "GPU device max available batch size: " << t << std::endl;
ASSERT_METRIC_SUPPORTED_IE(METRIC_KEY(MAX_BATCH_SIZE));
}
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassExecutableNetworkGetMetricTest, IEClassGetMetricTest_GPU_MAX_BATCH_SIZE_STREAM_DEVICE_MEM,
::testing::Values("GPU")
);
using IEClassGetMetricTest_GPU_UARCH_VERSION = BehaviorTestsUtils::IEClassBaseTestP;
TEST_P(IEClassGetMetricTest_GPU_UARCH_VERSION, GetMetricAndPrintNoThrow) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
@@ -153,6 +218,189 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values("GPU")
);
using IEClassGetMetricTest_GPU_MEMORY_STATISTICS_DEFAULT = BehaviorTestsUtils::IEClassBaseTestP;
TEST_P(IEClassGetMetricTest_GPU_MEMORY_STATISTICS_DEFAULT, GetMetricAndPrintNoThrow) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
InferenceEngine::Core ie;
InferenceEngine::Parameter p;
InferenceEngine::ExecutableNetwork exec_net = ie.LoadNetwork(simpleCnnNetwork, deviceName);
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t = p;
ASSERT_FALSE(t.empty());
std::cout << "Memory Statistics: " << std::endl;
for (auto &&kv : t) {
ASSERT_NE(kv.second, 0);
std::cout << kv.first << ": " << kv.second << " bytes" << std::endl;
}
ASSERT_METRIC_SUPPORTED_IE(GPU_METRIC_KEY(MEMORY_STATISTICS));
}
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassGetMetricTest, IEClassGetMetricTest_GPU_MEMORY_STATISTICS_DEFAULT,
::testing::Values("GPU")
);
using IEClassGetMetricTest_GPU_MEMORY_STATISTICS_MULTIPLE_NETWORKS = BehaviorTestsUtils::IEClassBaseTestP;
TEST_P(IEClassGetMetricTest_GPU_MEMORY_STATISTICS_MULTIPLE_NETWORKS, GetMetricAndPrintNoThrow) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
InferenceEngine::Core ie;
InferenceEngine::Parameter p;
InferenceEngine::ExecutableNetwork exec_net1 = ie.LoadNetwork(simpleCnnNetwork, deviceName);
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t1 = p;
ASSERT_FALSE(t1.empty());
for (auto &&kv : t1) {
ASSERT_NE(kv.second, 0);
}
InferenceEngine::ExecutableNetwork exec_net2 = ie.LoadNetwork(simpleCnnNetwork, deviceName);
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t2 = p;
ASSERT_FALSE(t2.empty());
for (auto &&kv : t2) {
ASSERT_NE(kv.second, 0);
auto iter = t1.find(kv.first);
if (iter != t1.end()) {
ASSERT_EQ(kv.second, t1[kv.first] * 2);
}
}
ASSERT_METRIC_SUPPORTED_IE(GPU_METRIC_KEY(MEMORY_STATISTICS));
}
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassGetMetricTest, IEClassGetMetricTest_GPU_MEMORY_STATISTICS_MULTIPLE_NETWORKS,
::testing::Values("GPU")
);
using IEClassGetMetricTest_GPU_MEMORY_STATISTICS_CHECK_VALUES = BehaviorTestsUtils::IEClassBaseTestP;
TEST_P(IEClassGetMetricTest_GPU_MEMORY_STATISTICS_CHECK_VALUES, GetMetricAndPrintNoThrow) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
InferenceEngine::Core ie;
InferenceEngine::Parameter p;
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t1 = p;
ASSERT_TRUE(t1.empty());
{
InferenceEngine::ExecutableNetwork exec_net1 = ie.LoadNetwork(simpleCnnNetwork, deviceName);
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t2 = p;
ASSERT_FALSE(t2.empty());
for (auto &&kv : t2) {
ASSERT_NE(kv.second, 0);
}
{
InferenceEngine::ExecutableNetwork exec_net2 = ie.LoadNetwork(actualCnnNetwork, deviceName);
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t3 = p;
ASSERT_FALSE(t3.empty());
for (auto &&kv : t3) {
ASSERT_NE(kv.second, 0);
}
}
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t4 = p;
ASSERT_FALSE(t4.empty());
for (auto &&kv : t4) {
ASSERT_NE(kv.second, 0);
if (kv.first.find("_cur") != std::string::npos) {
auto iter = t2.find(kv.first);
if (iter != t2.end()) {
ASSERT_EQ(t2[kv.first], kv.second);
}
}
}
}
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t5 = p;
ASSERT_FALSE(t5.empty());
for (auto &&kv : t5) {
if (kv.first.find("_cur") != std::string::npos) {
ASSERT_EQ(kv.second, 0);
}
}
ASSERT_METRIC_SUPPORTED_IE(GPU_METRIC_KEY(MEMORY_STATISTICS));
}
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassGetMetricTest, IEClassGetMetricTest_GPU_MEMORY_STATISTICS_CHECK_VALUES,
::testing::Values("GPU")
);
using IEClassGetMetricTest_GPU_MEMORY_STATISTICS_MULTI_THREADS = BehaviorTestsUtils::IEClassBaseTestP;
TEST_P(IEClassGetMetricTest_GPU_MEMORY_STATISTICS_MULTI_THREADS, GetMetricAndPrintNoThrow) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
InferenceEngine::Core ie;
InferenceEngine::Parameter p;
std::atomic<uint32_t> counter{0u};
std::vector<std::thread> threads(2);
// key: thread id, value: executable network
std::map<uint32_t, InferenceEngine::ExecutableNetwork> exec_net_map;
std::vector<InferenceEngine::CNNNetwork> networks;
networks.emplace_back(simpleCnnNetwork);
networks.emplace_back(simpleCnnNetwork);
InferenceEngine::ExecutableNetwork exec_net1 = ie.LoadNetwork(simpleCnnNetwork, deviceName);
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t1 = p;
ASSERT_FALSE(t1.empty());
for (auto &&kv : t1) {
ASSERT_NE(kv.second, 0);
}
for (auto & thread : threads) {
thread = std::thread([&](){
auto value = counter++;
exec_net_map[value] = ie.LoadNetwork(networks[value], deviceName);
});
}
for (auto & thread : threads) {
if (thread.joinable()) {
thread.join();
}
}
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, GPU_METRIC_KEY(MEMORY_STATISTICS)));
std::map<std::string, uint64_t> t2 = p;
ASSERT_FALSE(t2.empty());
for (auto &&kv : t2) {
ASSERT_NE(kv.second, 0);
auto iter = t1.find(kv.first);
if (iter != t1.end()) {
ASSERT_EQ(kv.second, t1[kv.first] * 3);
}
}
ASSERT_METRIC_SUPPORTED_IE(GPU_METRIC_KEY(MEMORY_STATISTICS));
}
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassGetMetricTest, IEClassGetMetricTest_GPU_MEMORY_STATISTICS_MULTI_THREADS,
::testing::Values("GPU")
);
//
// IE Class GetConfig
//
@@ -66,6 +66,7 @@ using OVClassGetMetricTest_OPTIMIZATION_CAPABILITIES = OVClassBaseTestP;
using OVClassGetMetricTest_DEVICE_GOPS = OVClassBaseTestP;
using OVClassGetMetricTest_DEVICE_TYPE = OVClassBaseTestP;
using OVClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS = OVClassBaseTestP;
using OVClassGetMetricTest_MAX_BATCH_SIZE = OVClassBaseTestP;
using OVClassGetMetricTest_ThrowUnsupported = OVClassBaseTestP;
using OVClassGetConfigTest = OVClassBaseTestP;
using OVClassGetConfigTest_ThrowUnsupported = OVClassBaseTestP;
@@ -578,6 +579,17 @@ TEST_P(OVClassGetMetricTest_OPTIMIZATION_CAPABILITIES, GetMetricAndPrintNoThrow)
OV_ASSERT_PROPERTY_SUPPORTED(ov::device::capabilities);
}
TEST_P(OVClassGetMetricTest_MAX_BATCH_SIZE, GetMetricAndPrintNoThrow) {
ov::Core ie;
uint32_t max_batch_size;
ASSERT_NO_THROW(max_batch_size = ie.get_property(deviceName, ov::max_batch_size));
std::cout << "Max batch size: " << max_batch_size << std::endl;
OV_ASSERT_PROPERTY_SUPPORTED(ov::max_batch_size);
}
TEST_P(OVClassGetMetricTest_DEVICE_GOPS, GetMetricAndPrintNoThrow) {
ov::Core ie = createCoreWithTemplate();
std::cout << "Device GOPS: " << std::endl;
@@ -82,8 +82,8 @@ def run(args):
if not args.perf_hint:
for device in devices:
supported_config_keys = benchmark.core.get_property(device, 'SUPPORTED_CONFIG_KEYS')
if 'PERFORMANCE_HINT' in supported_config_keys:
supported_properties = benchmark.core.get_property(device, 'SUPPORTED_PROPERTIES')
if 'PERFORMANCE_HINT' in supported_properties:
logger.warning(f"-hint default value is determined as 'THROUGHPUT' automatically for {device} device" +
"For more detailed information look at README.")
args.perf_hint = "throughput"
@@ -142,21 +142,29 @@ def run(args):
config[device]['PERFORMANCE_HINT_NUM_REQUESTS'] = str(args.number_infer_requests)
## the rest are individual per-device settings (overriding the values the device will deduce from perf hint)
def set_throughput_streams():
supported_properties = benchmark.core.get_property(device, 'SUPPORTED_PROPERTIES')
key = get_device_type_from_name(device) + "_THROUGHPUT_STREAMS"
if device in device_number_streams.keys():
## set to user defined value
supported_config_keys = benchmark.core.get_property(device, 'SUPPORTED_CONFIG_KEYS')
if key not in supported_config_keys:
if key in supported_properties:
config[device][key] = device_number_streams[device]
elif "NUM_STREAMS" in supported_properties:
key = "NUM_STREAMS"
config[device][key] = device_number_streams[device]
else:
raise Exception(f"Device {device} doesn't support config key '{key}'! " +
"Please specify -nstreams for correct devices in format <dev1>:<nstreams1>,<dev2>:<nstreams2>")
config[device][key] = device_number_streams[device]
elif key not in config[device].keys() and args.api_type == "async" and not is_flag_set_in_command_line('hint'):
## set the _AUTO value for the #streams
logger.warning(f"-nstreams default value is determined automatically for {device} device. " +
"Although the automatic selection usually provides a reasonable performance, "
"but it still may be non-optimal for some cases, for more information look at README.")
if device != MYRIAD_DEVICE_NAME: ## MYRIAD sets the default number of streams implicitly
config[device][key] = get_device_type_from_name(device) + "_THROUGHPUT_AUTO"
if key in supported_properties:
config[device][key] = get_device_type_from_name(device) + "_THROUGHPUT_AUTO"
elif "NUM_STREAMS" in supported_properties:
key = "NUM_STREAMS"
config[device][key] = "-1" # Set AUTO mode for streams number
if key in config[device].keys():
device_number_streams[device] = config[device][key]
@@ -316,19 +324,24 @@ def run(args):
next_step()
## actual device-deduced settings
for device in devices:
keys = benchmark.core.get_property(device, 'SUPPORTED_CONFIG_KEYS')
keys = benchmark.core.get_property(device, 'SUPPORTED_PROPERTIES')
logger.info(f'DEVICE: {device}')
for k in keys:
try:
logger.info(f' {k} , {benchmark.core.get_property(device, k)}')
except:
pass
if k not in ('SUPPORTED_METRICS', 'SUPPORTED_CONFIG_KEYS', 'SUPPORTED_PROPERTIES'):
try:
logger.info(f' {k} , {benchmark.core.get_property(device, k)}')
except:
logger.info(f' {k} , UNSUPPORTED TYPE ')
# Update number of streams
for device in device_number_streams.keys():
key = get_device_type_from_name(device) + '_THROUGHPUT_STREAMS'
device_number_streams[device] = benchmark.core.get_property(device, key)
try:
key = get_device_type_from_name(device) + '_THROUGHPUT_STREAMS'
device_number_streams[device] = benchmark.core.get_property(device, key)
except:
key = 'NUM_STREAMS'
device_number_streams[device] = benchmark.core.get_property(device, key)
# ------------------------------------ 9. Creating infer requests and preparing input data ----------------------
next_step()