From f45c846be04c41b6c94fea9988cc0e11b51137b0 Mon Sep 17 00:00:00 2001 From: Nadezhda Ageeva Date: Thu, 6 Oct 2022 13:54:25 +0400 Subject: [PATCH] [GPU] mark constants as supported on QN. QN helper. (#12990) * [GPU] mark constants as supported on QN. Fetch common part for CPU and GPU. * Code style * Apply review comments. Fix TEMPLATE plugin FP16, use common QN helper. * Apply review comments: CNNNetwork -> Model inside GetSupportedNodes * Apply review comments: move functino from plugin interface --- .../interface/ie_iplugin_internal.hpp | 36 ++- .../interface/ie_iplugin_internal.cpp | 86 ++++++- src/plugins/intel_cpu/src/plugin.cpp | 140 +++-------- .../include/intel_gpu/plugin/plugin.hpp | 2 +- src/plugins/intel_gpu/src/plugin/plugin.cpp | 228 ++++-------------- .../src/template_executable_network.cpp | 11 +- src/plugins/template/src/template_plugin.cpp | 106 +++----- 7 files changed, 231 insertions(+), 378 deletions(-) diff --git a/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp b/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp index af347051e54..d366ffb42a6 100644 --- a/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp +++ b/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp @@ -96,6 +96,31 @@ SetExeNetworkInfo(const std::shared_ptr& exeNetwork, const std::shared_ptr& function, bool new_api); +/** + * @brief Returns set of nodes which were removed after transformation. + * If originalFunction contains node1 and transformedFunction does not + * contains node1 in ops list, node1 will be returned. + * @param originalFunction Original network + * @param transformedFunction Transformed network + * @return Set of strings which contains removed node names + */ +INFERENCE_ENGINE_API_CPP(std::unordered_set) +GetRemovedNodes(const std::shared_ptr& originalFunction, + const std::shared_ptr& transformedFunction); + +/** + * @brief Returns set of nodes from original model which are + * determined as supported after applied transformation pipeline. + * @param model Original model + * @param transform Transformation pipeline function + * @param is_node_supported Function returning whether node is supported or not + * @return Set of strings which contains supported node names + */ +INFERENCE_ENGINE_API_CPP(std::unordered_set) +GetSupportedNodes(const std::shared_ptr& model, + std::function&)> transform, + std::function)> is_node_supported); + /** * @interface IInferencePlugin * @brief An API of plugin to be implemented by a plugin @@ -347,17 +372,6 @@ protected: void SetExeNetworkInfo(const std::shared_ptr& exeNetwork, const std::shared_ptr& function); - /** - * @brief Returns set of nodes which were removed after transformation. - * If originalFunction contains node1 and transformedFunction does not - * contains node1 in ops list, node1 will be returned. - * @param originalFunction Original network - * @param transformedFunction Transformed network - * @return Set of strings which contains removed node names - */ - std::unordered_set GetRemovedNodes(const std::shared_ptr& originalFunction, - const std::shared_ptr& transformedFunction) const; - std::string _pluginName; //!< A device name that plugins enables std::map _config; //!< A map config keys -> values std::weak_ptr _core; //!< A pointer to ICore interface diff --git a/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp b/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp index ffcb692cf80..dfd713701b6 100644 --- a/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp +++ b/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp @@ -310,9 +310,8 @@ void IInferencePlugin::SetExeNetworkInfo(const std::shared_ptrSetPointerToPlugin(shared_from_this()); } -std::unordered_set IInferencePlugin::GetRemovedNodes( - const std::shared_ptr& originalFunction, - const std::shared_ptr& transformedFunction) const { +std::unordered_set GetRemovedNodes(const std::shared_ptr& originalFunction, + const std::shared_ptr& transformedFunction) { std::unordered_set result = {}; std::unordered_set transformedNodeNames = {}; @@ -330,6 +329,87 @@ std::unordered_set IInferencePlugin::GetRemovedNodes( return result; } +std::unordered_set GetSupportedNodes( + const std::shared_ptr& model, + std::function&)> transform, + std::function)> is_node_supported) { + // Collect original operation names + std::unordered_set original_ops; + for (auto&& node : model->get_ops()) { + original_ops.emplace(node->get_friendly_name()); + } + + auto transformed_model = model->clone(); + transform(transformed_model); + auto ops = transformed_model->get_ordered_ops(); + + // Mark removed nodes as supported + std::unordered_set supported = GetRemovedNodes(model, transformed_model); + std::unordered_set unsupported; + + for (auto&& op : ops) { + bool is_supported = false; + bool is_checked = false; + if (InferenceEngine::details::contains(original_ops, op->get_friendly_name())) { + is_supported = is_node_supported(op); + is_checked = true; + if (is_supported) { + supported.emplace(op->get_friendly_name()); + } else { + unsupported.emplace(op->get_friendly_name()); + } + } + + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) { + if (InferenceEngine::details::contains(original_ops, fusedLayerName)) { + if (!is_checked) { + is_supported = is_node_supported(op); + is_checked = true; + } + if (is_supported) { + supported.emplace(fusedLayerName); + } else { + unsupported.emplace(fusedLayerName); + } + } + } + } + for (auto&& unsupportedNode : unsupported) { + supported.erase(unsupportedNode); + } + for (auto&& node : model->get_ops()) { + if (InferenceEngine::details::contains(supported, node->get_friendly_name())) { + for (auto&& inputNodeOutput : node->input_values()) { + if (ngraph::op::is_constant(inputNodeOutput.get_node()) || + ngraph::op::is_parameter(inputNodeOutput.get_node())) { + supported.emplace(inputNodeOutput.get_node()->get_friendly_name()); + } + } + for (auto&& outputs : node->outputs()) { + for (auto&& outputNodeInput : outputs.get_target_inputs()) { + if (ngraph::op::is_output(outputNodeInput.get_node())) { + supported.emplace(outputNodeInput.get_node()->get_friendly_name()); + } + } + } + } + + if (ngraph::op::is_constant(node) || ngraph::op::is_parameter(node)) { + if (!InferenceEngine::details::contains( + supported, + node->output(0).get_target_inputs().begin()->get_node()->get_friendly_name())) { + supported.erase(node->get_friendly_name()); + } + } else if (ngraph::op::is_output(node)) { + if (!InferenceEngine::details::contains(supported, + node->input_values().begin()->get_node()->get_friendly_name())) { + supported.erase(node->get_friendly_name()); + } + } + } + return supported; +} + void SetExeNetworkInfo(const std::shared_ptr& exeNetwork, const std::shared_ptr& function, bool new_api) { diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index d751f3acc37..ea28cc64228 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -664,12 +664,6 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr postSnippetsManager.run_passes(nGraphFunc); } -static void Transformation(CNNNetwork& clonedNetwork, const bool _enableLPT, const bool _enableBF16, const bool _enableSnippets, const bool isLegacyApi) { - auto nGraphFunc = clonedNetwork.getFunction(); - TransformationUpToCPUSpecificOpSet(nGraphFunc, _enableLPT, _enableBF16, _enableSnippets, isLegacyApi); - ConvertToCPUSpecificOpset(nGraphFunc); -} - static bool streamsSet(const std::map& config) { return config.count(PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS) || config.count(ov::num_streams.name()); @@ -1033,107 +1027,43 @@ QueryNetworkResult Engine::QueryNetwork(const CNNNetwork& network, const std::ma QueryNetworkResult res; WeightsSharing::Ptr fake_w_cache; - auto function = network.getFunction(); - if (function != nullptr) { - std::unordered_set originalOps; - for (auto&& node : function->get_ops()) { - originalOps.emplace(node->get_friendly_name()); + + // TODO: Clarify the behavior of SetConfig method. Skip eng_config or not? + Config conf = engConfig; + conf.readProperties(config); + + if (conf.enableDynamicBatch) { + conf.batchLimit = static_cast(network.getBatchSize()); + } + + const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE); + const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/ + || Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */; + const bool enableSnippets = !(conf.cache_dir.empty() || conf.enableDynamicBatch || (conf.enforceBF16 + && dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core))); + + auto model = network.getFunction(); + if (model == nullptr) { + IE_THROW() << "Only ngraph-based models are supported!"; + } + + auto supported = GetSupportedNodes(model, + [&](std::shared_ptr& model) { + TransformationUpToCPUSpecificOpSet(model, enableLPT, conf.enforceBF16, enableSnippets, isLegacyAPI()); + ConvertToCPUSpecificOpset(model); + }, + [&](const std::shared_ptr& op) { + std::unique_ptr ptr; + try { + ptr.reset(Node::factory().create(op, {dnnl::engine::kind::cpu, 0}, extensionManager, fake_w_cache)); + } catch (const InferenceEngine::Exception&) { + return false; } + return true; + }); - // TODO: Clarify the behavior of SetConfig method. Skip eng_config or not? - Config conf = engConfig; - conf.readProperties(config); - - if (conf.enableDynamicBatch) { - conf.batchLimit = static_cast(network.getBatchSize()); - } - - auto clonedNetwork = InferenceEngine::details::cloneNetwork(network); - auto clonnedFunction = clonedNetwork.getFunction(); - const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE); - const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/ - || Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */; - const bool enableSnippets = !(conf.cache_dir.empty() || conf.enableDynamicBatch || (conf.enforceBF16 - && dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core))); - Transformation(clonedNetwork, enableLPT, conf.enforceBF16, enableSnippets, isLegacyAPI()); - auto ops = clonnedFunction->get_ordered_ops(); - - //Mark removed nodes as supported - std::unordered_set supported = GetRemovedNodes(function, clonnedFunction);; - std::unordered_set unsupported; - - auto layerIsSupported = [&](const std::shared_ptr& op) { - std::unique_ptr ptr; - try { - ptr.reset(Node::factory().create(op, {dnnl::engine::kind::cpu, 0}, extensionManager, fake_w_cache)); - } catch (const InferenceEngine::Exception&) { - return false; - } - return true; - }; - - for (auto&& op : ops) { - bool isSupported = false; - bool wasNodeAlreadyChecked = false; - if (InferenceEngine::details::contains(originalOps, op->get_friendly_name())) { - isSupported = layerIsSupported(op); - wasNodeAlreadyChecked = true; - if (isSupported) { - supported.emplace(op->get_friendly_name()); - } else { - unsupported.emplace(op->get_friendly_name()); - } - } - - for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) { - if (InferenceEngine::details::contains(originalOps, fusedLayerName)) { - if (!wasNodeAlreadyChecked) { - isSupported = layerIsSupported(op); - wasNodeAlreadyChecked = true; - } - if (isSupported) { - supported.emplace(fusedLayerName); - } else { - unsupported.emplace(fusedLayerName); - } - } - } - } - for (auto&& unsupportedNode : unsupported) { - supported.erase(unsupportedNode); - } - for (auto&& node : function->get_ops()) { - if (InferenceEngine::details::contains(supported, node->get_friendly_name())) { - for (auto&& inputNodeOutput : node->input_values()) { - if (ngraph::op::is_constant(inputNodeOutput.get_node()) || ngraph::op::is_parameter(inputNodeOutput.get_node())) { - supported.emplace(inputNodeOutput.get_node()->get_friendly_name()); - } - } - for (auto&& outputs : node->outputs()) { - for (auto&& outputNodeInput : outputs.get_target_inputs()) { - if (ngraph::op::is_output(outputNodeInput.get_node())) { - supported.emplace(outputNodeInput.get_node()->get_friendly_name()); - } - } - } - } - - if (ngraph::op::is_constant(node) || ngraph::op::is_parameter(node)) { - if (!InferenceEngine::details::contains(supported, node->output(0).get_target_inputs().begin()->get_node()->get_friendly_name())) { - supported.erase(node->get_friendly_name()); - } - } else if (ngraph::op::is_output(node)) { - if (!InferenceEngine::details::contains(supported, node->input_values().begin()->get_node()->get_friendly_name())) { - supported.erase(node->get_friendly_name()); - } - } - } - - for (auto&& layerName : supported) { - res.supportedLayersMap.emplace(layerName, GetName()); - } - } else { - IE_CPU_PLUGIN_THROW() << "Only ngraph-based models are supported!"; + for (auto&& layerName : supported) { + res.supportedLayersMap.emplace(layerName, GetName()); } return res; diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp index 89bb2a599c8..bbbe7db84f3 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp @@ -35,7 +35,7 @@ class Plugin : public InferenceEngine::IInferencePlugin, cldnn::device_info GetDeviceInfo(const std::map &config) const; InferenceEngine::CNNNetwork CloneAndTransformNetwork(const InferenceEngine::CNNNetwork& network, const Config& config) const; - + void TransformNetwork(std::shared_ptr& model, const Config& config) const; std::map ConvertPerfHintsToConfig(const std::map& network_config, const Config& plugin_config) const; diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index 2117b31fc74..5189e88eeea 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -97,22 +97,26 @@ cldnn::device_info Plugin::GetDeviceInfo(const std::map& model, const Config& config) const { + OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "Plugin::TransformNetwork"); + auto deviceInfo = GetDeviceInfo(config.key_config_map); + TransformationsPipeline transformations(config, deviceInfo); + transformations.apply(model); +} + InferenceEngine::CNNNetwork Plugin::CloneAndTransformNetwork(const InferenceEngine::CNNNetwork& network, const Config& config) const { OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "Plugin::CloneAndTransformNetwork"); CNNNetwork clonedNetwork = InferenceEngine::details::cloneNetwork(network); - if (clonedNetwork.getFunction()) { - auto nGraphFunc = clonedNetwork.getFunction(); - auto deviceInfo = GetDeviceInfo(config.key_config_map); - TransformationsPipeline transformations(config, deviceInfo); - transformations.apply(nGraphFunc); + auto nGraphFunc = clonedNetwork.getFunction(); + if (nGraphFunc) { + TransformNetwork(nGraphFunc, config); + GPU_DEBUG_GET_INSTANCE(debug_config); + GPU_DEBUG_IF(!debug_config->dump_graphs.empty()) { + auto path_base = debug_config->dump_graphs + "/" + network.getName() + "_" + "transformed_func"; + ov::pass::Serialize(path_base + ".xml", path_base + ".bin").run_on_model(nGraphFunc); } - - GPU_DEBUG_GET_INSTANCE(debug_config); - GPU_DEBUG_IF(!debug_config->dump_graphs.empty()) { - auto path_base = debug_config->dump_graphs + "/" + network.getName() + "_" + "transformed_func"; - ov::pass::Serialize(path_base + ".xml", path_base + ".bin").run_on_model(clonedNetwork.getFunction()); } return clonedNetwork; } @@ -383,179 +387,51 @@ QueryNetworkResult Plugin::QueryNetwork(const CNNNetwork& network, AnyMap(), conf)); } Program prog(m_defaultContext->getImpl()->GetEngine(), conf); - auto function = network.getFunction(); - if (function == nullptr) { - IE_THROW() << "CNNetworkImpl representation is not supported anymore"; + bool dyn_shape_batch_found = false; + + auto model = network.getFunction(); + if (model == nullptr) { + IE_THROW() << "Only ngraph-based models are supported!"; } - std::unordered_set originalOpNames; - auto originalOps = function->get_ops(); - for (auto&& node : originalOps) { - originalOpNames.emplace(node->get_friendly_name()); - } + auto supported = GetSupportedNodes(model, + [&](std::shared_ptr& model) { + std::map shapes; + std::map> batch_dim; + dyn_shape_batch_found = prog.IsDynBatchModel(model, shapes, batch_dim); + TransformNetwork(model, conf); + }, + [&](std::shared_ptr node) { + if (node->is_dynamic()) { + if (!dyn_shape_batch_found) + return false; - auto clonedNetwork = CloneAndTransformNetwork(network, conf); - auto func = clonedNetwork.getFunction(); - auto ops = func->get_ordered_ops(); + auto pshape = node->get_output_partial_shape(0); + if (pshape.rank().is_dynamic()) + return false; - //Mark removed nodes as supported - std::unordered_set supported = GetRemovedNodes(function, func); - std::unordered_set unsupported; - - std::unordered_set supportedNotOriginal; - std::unordered_set unsupportedNotOriginal; - - std::vector> constants; - - std::map shapes; - std::map> batch_dim; - bool dyn_shape_batch_found = prog.IsDynBatchModel(func, shapes, batch_dim); - auto layerIsSupported = [&](std::shared_ptr node) { - if (node->is_dynamic()) { - if (!dyn_shape_batch_found) - return false; - - auto pshape = node->get_output_partial_shape(0); - if (pshape.rank().is_dynamic()) - return false; - - int dynCount = 0; - int64_t batch_idx = -1; - for (size_t i = 0; i < pshape.size(); i++) { - if (pshape[i].is_dynamic()) { - dynCount++; - if (batch_idx < 0) { - batch_idx = i; + int dynCount = 0; + int64_t batch_idx = -1; + for (size_t i = 0; i < pshape.size(); i++) { + if (pshape[i].is_dynamic()) { + dynCount++; + if (batch_idx < 0) { + batch_idx = i; + } } } + + if (dynCount != 1) + return false; // more than one dimension is dynamic + + int64_t max_batch = pshape[batch_idx].get_max_length(); + if (max_batch <= 1) + return false; + + return true; } - - if (dynCount != 1) - return false; // more than one dimension is dynamic - - int64_t max_batch = pshape[batch_idx].get_max_length(); - if (max_batch <= 1) - return false; - - return true; - } - if (ngraph::is_type(node) || - ngraph::is_type(node) || - ngraph::is_type(node)) { - return false; - } - if (ngraph::is_type(node)) { - constants.push_back(node); - return false; - } - return prog.IsOpSupported(network, node) || - ngraph::op::is_parameter(node) || - ngraph::op::is_output(node); - }; - - // Get ops after transformations and check if it's supported - // Transformations might lead to the situation when single node is merged to multiple operations, - // so we mark original op as supported only if all nodes that it was merged into are supported - for (auto&& op : ops) { - bool isSupported = layerIsSupported(op); - if (InferenceEngine::details::contains(originalOpNames, op->get_friendly_name())) { - if (isSupported) { - supported.emplace(op->get_friendly_name()); - } else { - unsupported.emplace(op->get_friendly_name()); - } - } else { - if (isSupported) { - supportedNotOriginal.emplace(op->get_friendly_name()); - } else { - unsupportedNotOriginal.emplace(op->get_friendly_name()); - } - } - - for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) { - if (InferenceEngine::details::contains(originalOpNames, fusedLayerName)) { - if (isSupported) { - supported.emplace(fusedLayerName); - } else { - unsupported.emplace(fusedLayerName); - } - } else { - if (isSupported) { - supportedNotOriginal.emplace(fusedLayerName); - } else { - unsupportedNotOriginal.emplace(fusedLayerName); - } - } - } - } - - for (auto&& layerName : unsupported) { - if (InferenceEngine::details::contains(supported, layerName)) { - supported.erase(layerName); - } - } - unsupported.clear(); - - for (auto&& layerName : unsupportedNotOriginal) { - if (InferenceEngine::details::contains(supportedNotOriginal, layerName)) { - supportedNotOriginal.erase(layerName); - } - } - unsupportedNotOriginal.clear(); - - // 1. Constants are marked as supported when all outputs can be offloaded to GPU - for (const auto& op : constants) { - bool is_supported = true; - - for (size_t i = 0; i < op->get_output_size(); i++) { - auto outTensors = op->get_output_target_inputs(i); - for (auto& t : outTensors) { - auto output = t.get_node(); - const auto& name = output->get_friendly_name(); - if (!InferenceEngine::details::contains(supported, name) && - !InferenceEngine::details::contains(supportedNotOriginal, name)) { - is_supported = false; - break; - } - } - } - if (is_supported) { - if (InferenceEngine::details::contains(originalOpNames, op->get_friendly_name())) - supported.emplace(op->get_friendly_name()); - for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) - if (InferenceEngine::details::contains(originalOpNames, fusedLayerName)) - supported.emplace(fusedLayerName); - } - } - - // Mark original constants/parameters/results ops as supported for each supported operation - // since rt_info doesn't contain names of constant that are removed during constant folding - for (auto&& node : originalOps) { - if (InferenceEngine::details::contains(supported, node->get_friendly_name())) { - for (auto&& inputNodeOutput : node->input_values()) { - if (ngraph::op::is_constant(inputNodeOutput.get_node()) || ngraph::op::is_parameter(inputNodeOutput.get_node())) { - supported.emplace(inputNodeOutput.get_node()->get_friendly_name()); - } - } - for (auto&& outputs : node->outputs()) { - for (auto&& outputNodeInput : outputs.get_target_inputs()) { - if (ngraph::op::is_output(outputNodeInput.get_node())) { - supported.emplace(outputNodeInput.get_node()->get_friendly_name()); - } - } - } - } - - if (ngraph::op::is_constant(node) || ngraph::op::is_parameter(node)) { - if (!InferenceEngine::details::contains(supported, node->output(0).get_target_inputs().begin()->get_node()->get_friendly_name())) { - supported.erase(node->get_friendly_name()); - } - } else if (ngraph::op::is_output(node)) { - if (!InferenceEngine::details::contains(supported, node->input_values().begin()->get_node()->get_friendly_name())) { - supported.erase(node->get_friendly_name()); - } - } - } + return prog.IsOpSupported(network, node); + }); for (auto&& layerName : supported) { res.supportedLayersMap.emplace(layerName, GetName()); diff --git a/src/plugins/template/src/template_executable_network.cpp b/src/plugins/template/src/template_executable_network.cpp index d40809d3270..a723f0991bd 100644 --- a/src/plugins/template/src/template_executable_network.cpp +++ b/src/plugins/template/src/template_executable_network.cpp @@ -98,17 +98,20 @@ TemplatePlugin::ExecutableNetwork::ExecutableNetwork(std::istream& model, // ! [executable_network:map_graph] // forward declaration -std::shared_ptr TransformNetwork(const std::shared_ptr& function, - const InferenceEngine::InputsDataMap& inputInfoMap, - const InferenceEngine::OutputsDataMap& outputsInfoMap); +void TransformNetwork(std::shared_ptr& function, + const InferenceEngine::InputsDataMap& inputInfoMap, + const InferenceEngine::OutputsDataMap& outputsInfoMap); void TemplatePlugin::ExecutableNetwork::CompileNetwork(const std::shared_ptr& function, const InferenceEngine::InputsDataMap& inputInfoMap, const InferenceEngine::OutputsDataMap& outputsInfoMap) { // TODO: perform actual graph compilation / mapping to backend graph representation / kernels + // clone network + _function = ngraph::clone_function(*function); + // apply plugins transformations - _function = TransformNetwork(function, inputInfoMap, outputsInfoMap); + TransformNetwork(_function, inputInfoMap, outputsInfoMap); // Generate backend specific blob mappings. For example Inference Engine uses not ngraph::Result nodes friendly name // as inference request output names but the name of the layer before. diff --git a/src/plugins/template/src/template_plugin.cpp b/src/plugins/template/src/template_plugin.cpp index 76c3dbc2182..ba76529638c 100644 --- a/src/plugins/template/src/template_plugin.cpp +++ b/src/plugins/template/src/template_plugin.cpp @@ -14,6 +14,8 @@ #include #include #include +#include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp" +#include #include #include @@ -54,13 +56,10 @@ Plugin::~Plugin() { // ! [plugin:transform_network] -std::shared_ptr TransformNetwork(const std::shared_ptr& function, - const InferenceEngine::InputsDataMap& inputInfoMap, - const InferenceEngine::OutputsDataMap& outputsInfoMap) { - // 1. Copy ngraph::Function first to apply some transformations which modify original ngraph::Function - auto transformedNetwork = ngraph::clone_function(*function); - - // 2. Perform common optimizations and device-specific transformations +void TransformNetwork(std::shared_ptr& function, + const InferenceEngine::InputsDataMap& inputInfoMap, + const InferenceEngine::OutputsDataMap& outputsInfoMap) { + // Perform common optimizations and device-specific transformations ngraph::pass::Manager passManager; // Example: register transformation to convert preprocessing information to graph nodes passManager.register_pass(inputInfoMap); @@ -87,11 +86,15 @@ std::shared_ptr TransformNetwork(const std::shared_ptrdisable(); + pass_config->disable(); + // After `run_passes`, we have the transformed function, where operations match device operations, // and we can create device backend-dependent graph - passManager.run_passes(transformedNetwork); - - return transformedNetwork; + passManager.run_passes(function); } // ! [plugin:transform_network] @@ -133,24 +136,21 @@ InferenceEngine::QueryNetworkResult Plugin::QueryNetwork(const InferenceEngine:: OV_ITT_SCOPED_TASK(itt::domains::TemplatePlugin, "Plugin::QueryNetwork"); Configuration fullConfig{config, _cfg, false}; - auto function = network.getFunction(); - // 1. First of all we should store initial input operation set - std::unordered_set originalOps; - std::map friendlyNameToType; - for (auto&& node : function->get_ops()) { - originalOps.emplace(node->get_friendly_name()); - friendlyNameToType[node->get_friendly_name()] = node->get_type_info(); + auto model = network.getFunction(); + if (model == nullptr) { + IE_THROW() << "Only ngraph-based models are supported!"; } - // 2. It is needed to apply all transformations as it is done in LoadExeNetworkImpl - auto transformedFunction = TransformNetwork(function, network.getInputsInfo(), network.getOutputsInfo()); - - // 3. The same input node can be transformed into supported and unsupported backend node - // So we need store as supported either unsupported node sets - std::unordered_set supported; - std::unordered_set unsupported; - ngraph::OpSet op_super_set; + auto supported = InferenceEngine::GetSupportedNodes( + model, + [&](std::shared_ptr& model) { + // 1. It is needed to apply all transformations as it is done in LoadExeNetworkImpl + TransformNetwork(model, network.getInputsInfo(), network.getOutputsInfo()); + }, + [&](std::shared_ptr node) { + // 2. Сheck whether node is supported + ngraph::OpSet op_super_set; #define _OPENVINO_OP_REG(NAME, NAMESPACE) op_super_set.insert(); #include "openvino/opsets/opset1_tbl.hpp" #include "openvino/opsets/opset2_tbl.hpp" @@ -161,60 +161,10 @@ InferenceEngine::QueryNetworkResult Plugin::QueryNetwork(const InferenceEngine:: #include "openvino/opsets/opset7_tbl.hpp" #include "openvino/opsets/opset8_tbl.hpp" #undef _OPENVINO_OP_REG - for (auto&& node : transformedFunction->get_ops()) { - // Extract transformation history from transformed node as list of nodes - for (auto&& fusedLayerName : ngraph::getFusedNamesVector(node)) { - // Filter just nodes from original operation set - // TODO: fill with actual decision rules based on whether kernel is supported by backend - if (InferenceEngine::details::contains(originalOps, fusedLayerName)) { - if (op_super_set.contains_type(friendlyNameToType[fusedLayerName])) { - supported.emplace(fusedLayerName); - } else { - unsupported.emplace(fusedLayerName); - } - } - } - } + return op_super_set.contains_type(node->get_type_info()); + }); - // 4. The result set should contain just nodes from supported set - for (auto&& unsupportedNode : unsupported) { - supported.erase(unsupportedNode); - } - - for (auto&& node : function->get_ops()) { - // 5. If some housekeeping nodes were not added - add them. - if (InferenceEngine::details::contains(supported, node->get_friendly_name())) { - for (auto&& inputNodeOutput : node->input_values()) { - if (ngraph::op::is_constant(inputNodeOutput.get_node()) || - ngraph::op::is_parameter(inputNodeOutput.get_node())) { - supported.emplace(inputNodeOutput.get_node()->get_friendly_name()); - } - } - for (auto&& outputs : node->outputs()) { - for (auto&& outputNodeInput : outputs.get_target_inputs()) { - if (ngraph::op::is_output(outputNodeInput.get_node())) { - supported.emplace(outputNodeInput.get_node()->get_friendly_name()); - } - } - } - } - - // 6. Eliminate subgraphs that consist of housekeeping nodes only - if (ngraph::op::is_constant(node) || ngraph::op::is_parameter(node)) { - if (!InferenceEngine::details::contains( - supported, - node->output(0).get_target_inputs().begin()->get_node()->get_friendly_name())) { - supported.erase(node->get_friendly_name()); - } - } else if (ngraph::op::is_output(node)) { - if (!InferenceEngine::details::contains(supported, - node->input_values().begin()->get_node()->get_friendly_name())) { - supported.erase(node->get_friendly_name()); - } - } - } - - // 7. Produce the result + // 3. Produce the result InferenceEngine::QueryNetworkResult res; for (auto&& layerName : supported) { res.supportedLayersMap.emplace(layerName, GetName());