IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GPU] mark constants as supported on QN. QN helper. (#12990)

Browse Source 
* [GPU] mark constants as supported on QN. Fetch common part for CPU and GPU.

* Code style

* Apply review comments. Fix TEMPLATE plugin FP16, use common QN helper.

* Apply review comments: CNNNetwork -> Model inside GetSupportedNodes

* Apply review comments: move functino from plugin interface
This commit is contained in:
Nadezhda Ageeva 2022-10-06 13:54:25 +04:00 committed by GitHub
parent 8a9c19e3eb
commit f45c846be0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 231 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp
View File

@ -96,6 +96,31 @@ SetExeNetworkInfo(const std::shared_ptr<IExecutableNetworkInternal>& exeNetwork,
const std::shared_ptr<const ov::Model>& function,
bool new_api);
/**
* @brief Returns set of nodes which were removed after transformation.
* If originalFunction contains node1 and transformedFunction does not
* contains node1 in ops list, node1 will be returned.
* @param originalFunction Original network
* @param transformedFunction Transformed network
* @return Set of strings which contains removed node names
*/
INFERENCE_ENGINE_API_CPP(std::unordered_set<std::string>)
GetRemovedNodes(const std::shared_ptr<const ov::Model>& originalFunction,
const std::shared_ptr<const ov::Model>& transformedFunction);
/**
* @brief Returns set of nodes from original model which are
* determined as supported after applied transformation pipeline.
* @param model Original model
* @param transform Transformation pipeline function
* @param is_node_supported Function returning whether node is supported or not
* @return Set of strings which contains supported node names
*/
INFERENCE_ENGINE_API_CPP(std::unordered_set<std::string>)
GetSupportedNodes(const std::shared_ptr<const ov::Model>& model,
std::function<void(std::shared_ptr<ov::Model>&)> transform,
std::function<bool(const std::shared_ptr<ngraph::Node>)> is_node_supported);
/**
* @interface IInferencePlugin
* @brief An API of plugin to be implemented by a plugin
@ -347,17 +372,6 @@ protected:
void SetExeNetworkInfo(const std::shared_ptr<IExecutableNetworkInternal>& exeNetwork,
const std::shared_ptr<const ov::Model>& function);
/**
* @brief Returns set of nodes which were removed after transformation.
* If originalFunction contains node1 and transformedFunction does not
* contains node1 in ops list, node1 will be returned.
* @param originalFunction Original network
* @param transformedFunction Transformed network
* @return Set of strings which contains removed node names
*/
std::unordered_set<std::string> GetRemovedNodes(const std::shared_ptr<const ov::Model>& originalFunction,
const std::shared_ptr<const ov::Model>& transformedFunction) const;
std::string _pluginName; //!< A device name that plugins enables
std::map<std::string, std::string> _config; //!< A map config keys -> values
std::weak_ptr<ov::ICore> _core; //!< A pointer to ICore interface

86
src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp
View File

@ -310,9 +310,8 @@ void IInferencePlugin::SetExeNetworkInfo(const std::shared_ptr<IExecutableNetwor
exeNetwork->SetPointerToPlugin(shared_from_this());
}
std::unordered_set<std::string> IInferencePlugin::GetRemovedNodes(
const std::shared_ptr<const ov::Model>& originalFunction,
const std::shared_ptr<const ov::Model>& transformedFunction) const {
std::unordered_set<std::string> GetRemovedNodes(const std::shared_ptr<const ov::Model>& originalFunction,
const std::shared_ptr<const ov::Model>& transformedFunction) {
std::unordered_set<std::string> result = {};
std::unordered_set<std::string> transformedNodeNames = {};
@ -330,6 +329,87 @@ std::unordered_set<std::string> IInferencePlugin::GetRemovedNodes(
return result;
}
std::unordered_set<std::string> GetSupportedNodes(
const std::shared_ptr<const ov::Model>& model,
std::function<void(std::shared_ptr<ov::Model>&)> transform,
std::function<bool(const std::shared_ptr<ngraph::Node>)> is_node_supported) {
// Collect original operation names
std::unordered_set<std::string> original_ops;
for (auto&& node : model->get_ops()) {
original_ops.emplace(node->get_friendly_name());
}
auto transformed_model = model->clone();
transform(transformed_model);
auto ops = transformed_model->get_ordered_ops();
// Mark removed nodes as supported
std::unordered_set<std::string> supported = GetRemovedNodes(model, transformed_model);
std::unordered_set<std::string> unsupported;
for (auto&& op : ops) {
bool is_supported = false;
bool is_checked = false;
if (InferenceEngine::details::contains(original_ops, op->get_friendly_name())) {
is_supported = is_node_supported(op);
is_checked = true;
if (is_supported) {
supported.emplace(op->get_friendly_name());
} else {
unsupported.emplace(op->get_friendly_name());
}
}
for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) {
if (InferenceEngine::details::contains(original_ops, fusedLayerName)) {
if (!is_checked) {
is_supported = is_node_supported(op);
is_checked = true;
}
if (is_supported) {
supported.emplace(fusedLayerName);
} else {
unsupported.emplace(fusedLayerName);
}
}
}
}
for (auto&& unsupportedNode : unsupported) {
supported.erase(unsupportedNode);
}
for (auto&& node : model->get_ops()) {
if (InferenceEngine::details::contains(supported, node->get_friendly_name())) {
for (auto&& inputNodeOutput : node->input_values()) {
if (ngraph::op::is_constant(inputNodeOutput.get_node()) ||
ngraph::op::is_parameter(inputNodeOutput.get_node())) {
supported.emplace(inputNodeOutput.get_node()->get_friendly_name());
}
}
for (auto&& outputs : node->outputs()) {
for (auto&& outputNodeInput : outputs.get_target_inputs()) {
if (ngraph::op::is_output(outputNodeInput.get_node())) {
supported.emplace(outputNodeInput.get_node()->get_friendly_name());
}
}
}
}
if (ngraph::op::is_constant(node) || ngraph::op::is_parameter(node)) {
if (!InferenceEngine::details::contains(
supported,
node->output(0).get_target_inputs().begin()->get_node()->get_friendly_name())) {
supported.erase(node->get_friendly_name());
}
} else if (ngraph::op::is_output(node)) {
if (!InferenceEngine::details::contains(supported,
node->input_values().begin()->get_node()->get_friendly_name())) {
supported.erase(node->get_friendly_name());
}
}
}
return supported;
}
void SetExeNetworkInfo(const std::shared_ptr<IExecutableNetworkInternal>& exeNetwork,
const std::shared_ptr<const ov::Model>& function,
bool new_api) {

140
src/plugins/intel_cpu/src/plugin.cpp
View File

@ -664,12 +664,6 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr<ngraph::Function>
postSnippetsManager.run_passes(nGraphFunc);
}
static void Transformation(CNNNetwork& clonedNetwork, const bool _enableLPT, const bool _enableBF16, const bool _enableSnippets, const bool isLegacyApi) {
auto nGraphFunc = clonedNetwork.getFunction();
TransformationUpToCPUSpecificOpSet(nGraphFunc, _enableLPT, _enableBF16, _enableSnippets, isLegacyApi);
ConvertToCPUSpecificOpset(nGraphFunc);
}
static bool streamsSet(const std::map<std::string, std::string>& config) {
return config.count(PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS) ||
config.count(ov::num_streams.name());
@ -1033,107 +1027,43 @@ QueryNetworkResult Engine::QueryNetwork(const CNNNetwork& network, const std::ma
QueryNetworkResult res;
WeightsSharing::Ptr fake_w_cache;
auto function = network.getFunction();
if (function != nullptr) {
std::unordered_set<std::string> originalOps;
for (auto&& node : function->get_ops()) {
originalOps.emplace(node->get_friendly_name());
// TODO: Clarify the behavior of SetConfig method. Skip eng_config or not?
Config conf = engConfig;
conf.readProperties(config);
if (conf.enableDynamicBatch) {
conf.batchLimit = static_cast<int>(network.getBatchSize());
}
const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE);
const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/
|| Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */;
const bool enableSnippets = !(conf.cache_dir.empty() || conf.enableDynamicBatch || (conf.enforceBF16
&& dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core)));
auto model = network.getFunction();
if (model == nullptr) {
IE_THROW() << "Only ngraph-based models are supported!";
}
auto supported = GetSupportedNodes(model,
[&](std::shared_ptr<ov::Model>& model) {
TransformationUpToCPUSpecificOpSet(model, enableLPT, conf.enforceBF16, enableSnippets, isLegacyAPI());
ConvertToCPUSpecificOpset(model);
},
[&](const std::shared_ptr<ngraph::Node>& op) {
std::unique_ptr<Node> ptr;
try {
ptr.reset(Node::factory().create(op, {dnnl::engine::kind::cpu, 0}, extensionManager, fake_w_cache));
} catch (const InferenceEngine::Exception&) {
return false;
}
return true;
});
// TODO: Clarify the behavior of SetConfig method. Skip eng_config or not?
Config conf = engConfig;
conf.readProperties(config);
if (conf.enableDynamicBatch) {
conf.batchLimit = static_cast<int>(network.getBatchSize());
}
auto clonedNetwork = InferenceEngine::details::cloneNetwork(network);
auto clonnedFunction = clonedNetwork.getFunction();
const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE);
const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/
|| Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */;
const bool enableSnippets = !(conf.cache_dir.empty() || conf.enableDynamicBatch || (conf.enforceBF16
&& dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core)));
Transformation(clonedNetwork, enableLPT, conf.enforceBF16, enableSnippets, isLegacyAPI());
auto ops = clonnedFunction->get_ordered_ops();
//Mark removed nodes as supported
std::unordered_set<std::string> supported = GetRemovedNodes(function, clonnedFunction);;
std::unordered_set<std::string> unsupported;
auto layerIsSupported = [&](const std::shared_ptr<ngraph::Node>& op) {
std::unique_ptr<Node> ptr;
try {
ptr.reset(Node::factory().create(op, {dnnl::engine::kind::cpu, 0}, extensionManager, fake_w_cache));
} catch (const InferenceEngine::Exception&) {
return false;
}
return true;
};
for (auto&& op : ops) {
bool isSupported = false;
bool wasNodeAlreadyChecked = false;
if (InferenceEngine::details::contains(originalOps, op->get_friendly_name())) {
isSupported = layerIsSupported(op);
wasNodeAlreadyChecked = true;
if (isSupported) {
supported.emplace(op->get_friendly_name());
} else {
unsupported.emplace(op->get_friendly_name());
}
}
for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) {
if (InferenceEngine::details::contains(originalOps, fusedLayerName)) {
if (!wasNodeAlreadyChecked) {
isSupported = layerIsSupported(op);
wasNodeAlreadyChecked = true;
}
if (isSupported) {
supported.emplace(fusedLayerName);
} else {
unsupported.emplace(fusedLayerName);
}
}
}
}
for (auto&& unsupportedNode : unsupported) {
supported.erase(unsupportedNode);
}
for (auto&& node : function->get_ops()) {
if (InferenceEngine::details::contains(supported, node->get_friendly_name())) {
for (auto&& inputNodeOutput : node->input_values()) {
if (ngraph::op::is_constant(inputNodeOutput.get_node()) || ngraph::op::is_parameter(inputNodeOutput.get_node())) {
supported.emplace(inputNodeOutput.get_node()->get_friendly_name());
}
}
for (auto&& outputs : node->outputs()) {
for (auto&& outputNodeInput : outputs.get_target_inputs()) {
if (ngraph::op::is_output(outputNodeInput.get_node())) {
supported.emplace(outputNodeInput.get_node()->get_friendly_name());
}
}
}
}
if (ngraph::op::is_constant(node) || ngraph::op::is_parameter(node)) {
if (!InferenceEngine::details::contains(supported, node->output(0).get_target_inputs().begin()->get_node()->get_friendly_name())) {
supported.erase(node->get_friendly_name());
}
} else if (ngraph::op::is_output(node)) {
if (!InferenceEngine::details::contains(supported, node->input_values().begin()->get_node()->get_friendly_name())) {
supported.erase(node->get_friendly_name());
}
}
}
for (auto&& layerName : supported) {
res.supportedLayersMap.emplace(layerName, GetName());
}
} else {
IE_CPU_PLUGIN_THROW() << "Only ngraph-based models are supported!";
for (auto&& layerName : supported) {
res.supportedLayersMap.emplace(layerName, GetName());
}
return res;

2
src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp
View File

@ -35,7 +35,7 @@ class Plugin : public InferenceEngine::IInferencePlugin,
cldnn::device_info GetDeviceInfo(const std::map<std::string, std::string> &config) const;
InferenceEngine::CNNNetwork CloneAndTransformNetwork(const InferenceEngine::CNNNetwork& network,
const Config& config) const;
void TransformNetwork(std::shared_ptr<ov::Model>& model, const Config& config) const;
std::map<std::string, std::string> ConvertPerfHintsToConfig(const std::map<std::string, std::string>& network_config,
const Config& plugin_config) const;

228
src/plugins/intel_gpu/src/plugin/plugin.cpp
View File

@ -97,22 +97,26 @@ cldnn::device_info Plugin::GetDeviceInfo(const std::map<std::string, std::string
return device_info;
}
void Plugin::TransformNetwork(std::shared_ptr<ov::Model>& model, const Config& config) const {
OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "Plugin::TransformNetwork");
auto deviceInfo = GetDeviceInfo(config.key_config_map);
TransformationsPipeline transformations(config, deviceInfo);
transformations.apply(model);
}
InferenceEngine::CNNNetwork Plugin::CloneAndTransformNetwork(const InferenceEngine::CNNNetwork& network,
const Config& config) const {
OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "Plugin::CloneAndTransformNetwork");
CNNNetwork clonedNetwork = InferenceEngine::details::cloneNetwork(network);
if (clonedNetwork.getFunction()) {
auto nGraphFunc = clonedNetwork.getFunction();
auto deviceInfo = GetDeviceInfo(config.key_config_map);
TransformationsPipeline transformations(config, deviceInfo);
transformations.apply(nGraphFunc);
auto nGraphFunc = clonedNetwork.getFunction();
if (nGraphFunc) {
TransformNetwork(nGraphFunc, config);
GPU_DEBUG_GET_INSTANCE(debug_config);
GPU_DEBUG_IF(!debug_config->dump_graphs.empty()) {
auto path_base = debug_config->dump_graphs + "/" + network.getName() + "_" + "transformed_func";
ov::pass::Serialize(path_base + ".xml", path_base + ".bin").run_on_model(nGraphFunc);
}
GPU_DEBUG_GET_INSTANCE(debug_config);
GPU_DEBUG_IF(!debug_config->dump_graphs.empty()) {
auto path_base = debug_config->dump_graphs + "/" + network.getName() + "_" + "transformed_func";
ov::pass::Serialize(path_base + ".xml", path_base + ".bin").run_on_model(clonedNetwork.getFunction());
}
return clonedNetwork;
}
@ -383,179 +387,51 @@ QueryNetworkResult Plugin::QueryNetwork(const CNNNetwork& network,
AnyMap(), conf));
}
Program prog(m_defaultContext->getImpl()->GetEngine(), conf);
auto function = network.getFunction();
if (function == nullptr) {
IE_THROW() << "CNNetworkImpl representation is not supported anymore";
bool dyn_shape_batch_found = false;
auto model = network.getFunction();
if (model == nullptr) {
IE_THROW() << "Only ngraph-based models are supported!";
}
std::unordered_set<std::string> originalOpNames;
auto originalOps = function->get_ops();
for (auto&& node : originalOps) {
originalOpNames.emplace(node->get_friendly_name());
}
auto supported = GetSupportedNodes(model,
[&](std::shared_ptr<ov::Model>& model) {
std::map<std::string, ngraph::PartialShape> shapes;
std::map<std::string, std::pair<int64_t, int64_t>> batch_dim;
dyn_shape_batch_found = prog.IsDynBatchModel(model, shapes, batch_dim);
TransformNetwork(model, conf);
},
[&](std::shared_ptr<ngraph::Node> node) {
if (node->is_dynamic()) {
if (!dyn_shape_batch_found)
return false;
auto clonedNetwork = CloneAndTransformNetwork(network, conf);
auto func = clonedNetwork.getFunction();
auto ops = func->get_ordered_ops();
auto pshape = node->get_output_partial_shape(0);
if (pshape.rank().is_dynamic())
return false;
//Mark removed nodes as supported
std::unordered_set<std::string> supported = GetRemovedNodes(function, func);
std::unordered_set<std::string> unsupported;
std::unordered_set<std::string> supportedNotOriginal;
std::unordered_set<std::string> unsupportedNotOriginal;
std::vector<std::shared_ptr<ngraph::Node>> constants;
std::map<std::string, ngraph::PartialShape> shapes;
std::map<std::string, std::pair<int64_t, int64_t>> batch_dim;
bool dyn_shape_batch_found = prog.IsDynBatchModel(func, shapes, batch_dim);
auto layerIsSupported = [&](std::shared_ptr<ngraph::Node> node) {
if (node->is_dynamic()) {
if (!dyn_shape_batch_found)
return false;
auto pshape = node->get_output_partial_shape(0);
if (pshape.rank().is_dynamic())
return false;
int dynCount = 0;
int64_t batch_idx = -1;
for (size_t i = 0; i < pshape.size(); i++) {
if (pshape[i].is_dynamic()) {
dynCount++;
if (batch_idx < 0) {
batch_idx = i;
int dynCount = 0;
int64_t batch_idx = -1;
for (size_t i = 0; i < pshape.size(); i++) {
if (pshape[i].is_dynamic()) {
dynCount++;
if (batch_idx < 0) {
batch_idx = i;
}
}
}
if (dynCount != 1)
return false; // more than one dimension is dynamic
int64_t max_batch = pshape[batch_idx].get_max_length();
if (max_batch <= 1)
return false;
return true;
}
if (dynCount != 1)
return false; // more than one dimension is dynamic
int64_t max_batch = pshape[batch_idx].get_max_length();
if (max_batch <= 1)
return false;
return true;
}
if (ngraph::is_type<const ngraph::op::v0::PriorBox>(node) ||
ngraph::is_type<const ngraph::op::v0::PriorBoxClustered>(node) ||
ngraph::is_type<const ngraph::op::v0::Proposal>(node)) {
return false;
}
if (ngraph::is_type<const ngraph::op::v0::Constant>(node)) {
constants.push_back(node);
return false;
}
return prog.IsOpSupported(network, node) ||
ngraph::op::is_parameter(node) ||
ngraph::op::is_output(node);
};
// Get ops after transformations and check if it's supported
// Transformations might lead to the situation when single node is merged to multiple operations,
// so we mark original op as supported only if all nodes that it was merged into are supported
for (auto&& op : ops) {
bool isSupported = layerIsSupported(op);
if (InferenceEngine::details::contains(originalOpNames, op->get_friendly_name())) {
if (isSupported) {
supported.emplace(op->get_friendly_name());
} else {
unsupported.emplace(op->get_friendly_name());
}
} else {
if (isSupported) {
supportedNotOriginal.emplace(op->get_friendly_name());
} else {
unsupportedNotOriginal.emplace(op->get_friendly_name());
}
}
for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) {
if (InferenceEngine::details::contains(originalOpNames, fusedLayerName)) {
if (isSupported) {
supported.emplace(fusedLayerName);
} else {
unsupported.emplace(fusedLayerName);
}
} else {
if (isSupported) {
supportedNotOriginal.emplace(fusedLayerName);
} else {
unsupportedNotOriginal.emplace(fusedLayerName);
}
}
}
}
for (auto&& layerName : unsupported) {
if (InferenceEngine::details::contains(supported, layerName)) {
supported.erase(layerName);
}
}
unsupported.clear();
for (auto&& layerName : unsupportedNotOriginal) {
if (InferenceEngine::details::contains(supportedNotOriginal, layerName)) {
supportedNotOriginal.erase(layerName);
}
}
unsupportedNotOriginal.clear();
// 1. Constants are marked as supported when all outputs can be offloaded to GPU
for (const auto& op : constants) {
bool is_supported = true;
for (size_t i = 0; i < op->get_output_size(); i++) {
auto outTensors = op->get_output_target_inputs(i);
for (auto& t : outTensors) {
auto output = t.get_node();
const auto& name = output->get_friendly_name();
if (!InferenceEngine::details::contains(supported, name) &&
!InferenceEngine::details::contains(supportedNotOriginal, name)) {
is_supported = false;
break;
}
}
}
if (is_supported) {
if (InferenceEngine::details::contains(originalOpNames, op->get_friendly_name()))
supported.emplace(op->get_friendly_name());
for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op))
if (InferenceEngine::details::contains(originalOpNames, fusedLayerName))
supported.emplace(fusedLayerName);
}
}
// Mark original constants/parameters/results ops as supported for each supported operation
// since rt_info doesn't contain names of constant that are removed during constant folding
for (auto&& node : originalOps) {
if (InferenceEngine::details::contains(supported, node->get_friendly_name())) {
for (auto&& inputNodeOutput : node->input_values()) {
if (ngraph::op::is_constant(inputNodeOutput.get_node()) || ngraph::op::is_parameter(inputNodeOutput.get_node())) {
supported.emplace(inputNodeOutput.get_node()->get_friendly_name());
}
}
for (auto&& outputs : node->outputs()) {
for (auto&& outputNodeInput : outputs.get_target_inputs()) {
if (ngraph::op::is_output(outputNodeInput.get_node())) {
supported.emplace(outputNodeInput.get_node()->get_friendly_name());
}
}
}
}
if (ngraph::op::is_constant(node) || ngraph::op::is_parameter(node)) {
if (!InferenceEngine::details::contains(supported, node->output(0).get_target_inputs().begin()->get_node()->get_friendly_name())) {
supported.erase(node->get_friendly_name());
}
} else if (ngraph::op::is_output(node)) {
if (!InferenceEngine::details::contains(supported, node->input_values().begin()->get_node()->get_friendly_name())) {
supported.erase(node->get_friendly_name());
}
}
}
return prog.IsOpSupported(network, node);
});
for (auto&& layerName : supported) {
res.supportedLayersMap.emplace(layerName, GetName());

11
src/plugins/template/src/template_executable_network.cpp
View File

@ -98,17 +98,20 @@ TemplatePlugin::ExecutableNetwork::ExecutableNetwork(std::istream& model,
// ! [executable_network:map_graph]
// forward declaration
std::shared_ptr<ngraph::Function> TransformNetwork(const std::shared_ptr<const ngraph::Function>& function,
const InferenceEngine::InputsDataMap& inputInfoMap,
const InferenceEngine::OutputsDataMap& outputsInfoMap);
void TransformNetwork(std::shared_ptr<ngraph::Function>& function,
const InferenceEngine::InputsDataMap& inputInfoMap,
const InferenceEngine::OutputsDataMap& outputsInfoMap);
void TemplatePlugin::ExecutableNetwork::CompileNetwork(const std::shared_ptr<const ngraph::Function>& function,
const InferenceEngine::InputsDataMap& inputInfoMap,
const InferenceEngine::OutputsDataMap& outputsInfoMap) {
// TODO: perform actual graph compilation / mapping to backend graph representation / kernels
// clone network
_function = ngraph::clone_function(*function);
// apply plugins transformations
_function = TransformNetwork(function, inputInfoMap, outputsInfoMap);
TransformNetwork(_function, inputInfoMap, outputsInfoMap);
// Generate backend specific blob mappings. For example Inference Engine uses not ngraph::Result nodes friendly name
// as inference request output names but the name of the layer before.

106
src/plugins/template/src/template_plugin.cpp
View File

@ -14,6 +14,8 @@
#include <ngraph/pass/manager.hpp>
#include <ngraph/opsets/opset.hpp>
#include <transformations/common_optimizations/common_optimizations.hpp>
#include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp"
#include <transformations/disable_decompression_convert_constant_folding.hpp>
#include <transformations/rt_info/fused_names_attribute.hpp>
#include <transformations/convert_precision.hpp>
@ -54,13 +56,10 @@ Plugin::~Plugin() {
// ! [plugin:transform_network]
std::shared_ptr<ngraph::Function> TransformNetwork(const std::shared_ptr<const ngraph::Function>& function,
const InferenceEngine::InputsDataMap& inputInfoMap,
const InferenceEngine::OutputsDataMap& outputsInfoMap) {
// 1. Copy ngraph::Function first to apply some transformations which modify original ngraph::Function
auto transformedNetwork = ngraph::clone_function(*function);
// 2. Perform common optimizations and device-specific transformations
void TransformNetwork(std::shared_ptr<ngraph::Function>& function,
const InferenceEngine::InputsDataMap& inputInfoMap,
const InferenceEngine::OutputsDataMap& outputsInfoMap) {
// Perform common optimizations and device-specific transformations
ngraph::pass::Manager passManager;
// Example: register transformation to convert preprocessing information to graph nodes
passManager.register_pass<ngraph::pass::AddPreprocessing>(inputInfoMap);
@ -87,11 +86,15 @@ std::shared_ptr<ngraph::Function> TransformNetwork(const std::shared_ptr<const n
// Register any other transformations
// ..
const auto& pass_config = passManager.get_pass_config();
// Allow FP16 Converts to be folded and FP16 constants to be upgraded to FP32 data type
pass_config->disable<ov::pass::DisableDecompressionConvertConstantFolding>();
pass_config->disable<ov::pass::ConvertCompressedOnlyToLegacy>();
// After `run_passes`, we have the transformed function, where operations match device operations,
// and we can create device backend-dependent graph
passManager.run_passes(transformedNetwork);
return transformedNetwork;
passManager.run_passes(function);
}
// ! [plugin:transform_network]
@ -133,24 +136,21 @@ InferenceEngine::QueryNetworkResult Plugin::QueryNetwork(const InferenceEngine::
OV_ITT_SCOPED_TASK(itt::domains::TemplatePlugin, "Plugin::QueryNetwork");
Configuration fullConfig{config, _cfg, false};
auto function = network.getFunction();
// 1. First of all we should store initial input operation set
std::unordered_set<std::string> originalOps;
std::map<std::string, ngraph::NodeTypeInfo> friendlyNameToType;
for (auto&& node : function->get_ops()) {
originalOps.emplace(node->get_friendly_name());
friendlyNameToType[node->get_friendly_name()] = node->get_type_info();
auto model = network.getFunction();
if (model == nullptr) {
IE_THROW() << "Only ngraph-based models are supported!";
}
// 2. It is needed to apply all transformations as it is done in LoadExeNetworkImpl
auto transformedFunction = TransformNetwork(function, network.getInputsInfo(), network.getOutputsInfo());
// 3. The same input node can be transformed into supported and unsupported backend node
// So we need store as supported either unsupported node sets
std::unordered_set<std::string> supported;
std::unordered_set<std::string> unsupported;
ngraph::OpSet op_super_set;
auto supported = InferenceEngine::GetSupportedNodes(
model,
[&](std::shared_ptr<ov::Model>& model) {
// 1. It is needed to apply all transformations as it is done in LoadExeNetworkImpl
TransformNetwork(model, network.getInputsInfo(), network.getOutputsInfo());
},
[&](std::shared_ptr<ngraph::Node> node) {
// 2. Сheck whether node is supported
ngraph::OpSet op_super_set;
#define _OPENVINO_OP_REG(NAME, NAMESPACE) op_super_set.insert<NAMESPACE::NAME>();
#include "openvino/opsets/opset1_tbl.hpp"
#include "openvino/opsets/opset2_tbl.hpp"
@ -161,60 +161,10 @@ InferenceEngine::QueryNetworkResult Plugin::QueryNetwork(const InferenceEngine::
#include "openvino/opsets/opset7_tbl.hpp"
#include "openvino/opsets/opset8_tbl.hpp"
#undef _OPENVINO_OP_REG
for (auto&& node : transformedFunction->get_ops()) {
// Extract transformation history from transformed node as list of nodes
for (auto&& fusedLayerName : ngraph::getFusedNamesVector(node)) {
// Filter just nodes from original operation set
// TODO: fill with actual decision rules based on whether kernel is supported by backend
if (InferenceEngine::details::contains(originalOps, fusedLayerName)) {
if (op_super_set.contains_type(friendlyNameToType[fusedLayerName])) {
supported.emplace(fusedLayerName);
} else {
unsupported.emplace(fusedLayerName);
}
}
}
}
return op_super_set.contains_type(node->get_type_info());
});
// 4. The result set should contain just nodes from supported set
for (auto&& unsupportedNode : unsupported) {
supported.erase(unsupportedNode);
}
for (auto&& node : function->get_ops()) {
// 5. If some housekeeping nodes were not added - add them.
if (InferenceEngine::details::contains(supported, node->get_friendly_name())) {
for (auto&& inputNodeOutput : node->input_values()) {
if (ngraph::op::is_constant(inputNodeOutput.get_node()) ||
ngraph::op::is_parameter(inputNodeOutput.get_node())) {
supported.emplace(inputNodeOutput.get_node()->get_friendly_name());
}
}
for (auto&& outputs : node->outputs()) {
for (auto&& outputNodeInput : outputs.get_target_inputs()) {
if (ngraph::op::is_output(outputNodeInput.get_node())) {
supported.emplace(outputNodeInput.get_node()->get_friendly_name());
}
}
}
}
// 6. Eliminate subgraphs that consist of housekeeping nodes only
if (ngraph::op::is_constant(node) || ngraph::op::is_parameter(node)) {
if (!InferenceEngine::details::contains(
supported,
node->output(0).get_target_inputs().begin()->get_node()->get_friendly_name())) {
supported.erase(node->get_friendly_name());
}
} else if (ngraph::op::is_output(node)) {
if (!InferenceEngine::details::contains(supported,
node->input_values().begin()->get_node()->get_friendly_name())) {
supported.erase(node->get_friendly_name());
}
}
}
// 7. Produce the result
// 3. Produce the result
InferenceEngine::QueryNetworkResult res;
for (auto&& layerName : supported) {
res.supportedLayersMap.emplace(layerName, GetName());