From 88e20199f02b00f422eddf5eb766d59f0aae1bfb Mon Sep 17 00:00:00 2001 From: Oleg Pipikin Date: Thu, 31 Mar 2022 07:24:46 +0300 Subject: [PATCH] Fix query network for hetero plugin (#10556) * Fix query network for hetero plugin * Apply comments * Fix1 * Add tests * Apply comments 2 * Apply comments 3 --- .../interface/ie_iplugin_internal.hpp | 11 + .../interface/ie_iplugin_internal.cpp | 21 + src/plugins/intel_cpu/src/plugin.cpp | 45 +- src/plugins/intel_gpu/src/plugin/plugin.cpp | 61 ++- .../common/src/ngraph/query_network.cpp | 19 +- .../myriad_plugin/myriad_plugin.cpp | 7 + .../cpu/behavior/hetero_cpu_query_network.cpp | 22 + .../gpu/behavior/hetero_gpu_query_network.cpp | 22 + .../behavior/plugin/hetero_query_network.hpp | 466 ++++++++++++++++++ 9 files changed, 645 insertions(+), 29 deletions(-) create mode 100644 src/tests/functional/plugin/cpu/behavior/hetero_cpu_query_network.cpp create mode 100644 src/tests/functional/plugin/gpu/behavior/hetero_gpu_query_network.cpp create mode 100644 src/tests/functional/plugin/shared/include/behavior/plugin/hetero_query_network.hpp diff --git a/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp b/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp index e164a65c70d..d476724fb68 100644 --- a/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp +++ b/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp @@ -335,6 +335,17 @@ protected: void SetExeNetworkInfo(const std::shared_ptr& exeNetwork, const std::shared_ptr& function); + /** + * @brief Returns set of nodes which were removed after transformation. + * If originalFunction contains node1 and transformedFunction does not + * contains node1 in ops list, node1 will be returned. + * @param originalFunction Original network + * @param transformedFunction Transformed network + * @return Set of strings which contains removed node names + */ + std::unordered_set GetRemovedNodes(const std::shared_ptr& originalFunction, + const std::shared_ptr& transformedFunction) const; + std::string _pluginName; //!< A device name that plugins enables std::map _config; //!< A map config keys -> values std::weak_ptr _core; //!< A pointer to ICore interface diff --git a/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp b/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp index 92ac8e919f6..8fbccabb034 100644 --- a/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp +++ b/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp @@ -20,6 +20,7 @@ #include "cnn_network_ngraph_impl.hpp" #include "cpp/ie_cnn_network.h" #include "exec_graph_info.hpp" +#include "ie_algorithm.hpp" #include "ie_api.h" #include "ie_icore.hpp" #include "ie_iextension.h" @@ -300,6 +301,26 @@ void IInferencePlugin::SetExeNetworkInfo(const std::shared_ptrSetPointerToPlugin(shared_from_this()); } +std::unordered_set IInferencePlugin::GetRemovedNodes( + const std::shared_ptr& originalFunction, + const std::shared_ptr& transformedFunction) const { + std::unordered_set result = {}; + std::unordered_set transformedNodeNames = {}; + + for (auto&& node : transformedFunction->get_ops()) { + transformedNodeNames.emplace(node->get_friendly_name()); + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(node)) + transformedNodeNames.emplace(fusedLayerName); + } + + for (auto&& originalNode : originalFunction->get_ops()) { + if (!InferenceEngine::details::contains(transformedNodeNames, originalNode->get_friendly_name())) + result.emplace(originalNode->get_friendly_name()); + } + + return result; +} + void SetExeNetworkInfo(const std::shared_ptr& exeNetwork, const std::shared_ptr& function, bool new_api) { diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index d9369bc56b3..d5ab9fafc64 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -915,28 +915,49 @@ QueryNetworkResult Engine::QueryNetwork(const CNNNetwork& network, const std::ma } auto clonedNetwork = InferenceEngine::details::cloneNetwork(network); + auto clonnedFunction = clonedNetwork.getFunction(); const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE); const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/ || Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */; const bool enableSnippets = !(conf.cache_dir.empty() || conf.enableDynamicBatch || (conf.enforceBF16 && dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core))); Transformation(clonedNetwork, enableLPT, enableSnippets, isLegacyAPI()); - auto ops = clonedNetwork.getFunction()->get_ordered_ops(); - std::unordered_set supported; + auto ops = clonnedFunction->get_ordered_ops(); + + //Mark removed nodes as supported + std::unordered_set supported = GetRemovedNodes(function, clonnedFunction);; std::unordered_set unsupported; - for (auto op : ops) { - auto layerIsSupported = [&] { - std::unique_ptr ptr; - try { - ptr.reset(Node::factory().create(op, {mkldnn::engine::kind::cpu, 0}, extensionManager, fake_w_cache)); - } catch (InferenceEngine::Exception&) { - return false; + + auto layerIsSupported = [&](const std::shared_ptr& op) { + std::unique_ptr ptr; + try { + ptr.reset(Node::factory().create(op, {mkldnn::engine::kind::cpu, 0}, extensionManager, fake_w_cache)); + } catch (const InferenceEngine::Exception&) { + return false; + } + return true; + }; + + for (auto&& op : ops) { + bool isSupported = false; + bool wasNodeAlreadyChecked = false; + if (InferenceEngine::details::contains(originalOps, op->get_friendly_name())) { + isSupported = layerIsSupported(op); + wasNodeAlreadyChecked = true; + if (isSupported) { + supported.emplace(op->get_friendly_name()); + } else { + unsupported.emplace(op->get_friendly_name()); } - return true; - } (); + } + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) { if (InferenceEngine::details::contains(originalOps, fusedLayerName)) { - if (layerIsSupported) { + if (!wasNodeAlreadyChecked) { + isSupported = layerIsSupported(op); + wasNodeAlreadyChecked = true; + } + if (isSupported) { supported.emplace(fusedLayerName); } else { unsupported.emplace(fusedLayerName); diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index 44eb01608ee..cc87261c3e0 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -398,10 +398,14 @@ QueryNetworkResult Plugin::QueryNetwork(const CNNNetwork& network, auto clonedNetwork = CloneAndTransformNetwork(network, conf); auto func = clonedNetwork.getFunction(); auto ops = func->get_ordered_ops(); - std::unordered_set supported; + + //Mark removed nodes as supported + std::unordered_set supported = GetRemovedNodes(function, func);; std::unordered_set unsupported; - std::unordered_set constantsNames; + std::unordered_set supportedNotOriginal; + std::unordered_set unsupportedNotOriginal; + std::vector> constants; std::map shapes; @@ -442,34 +446,46 @@ QueryNetworkResult Plugin::QueryNetwork(const CNNNetwork& network, return false; } if (ngraph::is_type(node)) { - constantsNames.emplace(node->get_friendly_name()); constants.push_back(node); return false; } - return prog.IsOpSupported(network, node) && - !ngraph::op::is_parameter(node) && - !ngraph::op::is_output(node); + return prog.IsOpSupported(network, node) || + ngraph::op::is_parameter(node) || + ngraph::op::is_output(node); }; // Get ops after transformations and check if it's supported // Transformations might lead to the situation when single node is merged to multiple operations, // so we mark original op as supported only if all nodes that it was merged into are supported - bool wasNodeAlreadyChecked = false; - bool isSupported = false; for (auto&& op : ops) { - wasNodeAlreadyChecked = false; - isSupported = false; + bool isSupported = layerIsSupported(op); + if (InferenceEngine::details::contains(originalOpNames, op->get_friendly_name())) { + if (isSupported) { + supported.emplace(op->get_friendly_name()); + } else { + unsupported.emplace(op->get_friendly_name()); + } + } else { + if (isSupported) { + supportedNotOriginal.emplace(op->get_friendly_name()); + } else { + unsupportedNotOriginal.emplace(op->get_friendly_name()); + } + } + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) { if (InferenceEngine::details::contains(originalOpNames, fusedLayerName)) { - if (!wasNodeAlreadyChecked) { - isSupported = layerIsSupported(op); - wasNodeAlreadyChecked = true; - } if (isSupported) { supported.emplace(fusedLayerName); } else { unsupported.emplace(fusedLayerName); } + } else { + if (isSupported) { + supportedNotOriginal.emplace(fusedLayerName); + } else { + unsupportedNotOriginal.emplace(fusedLayerName); + } } } } @@ -481,22 +497,35 @@ QueryNetworkResult Plugin::QueryNetwork(const CNNNetwork& network, } unsupported.clear(); + for (auto&& layerName : unsupportedNotOriginal) { + if (InferenceEngine::details::contains(supportedNotOriginal, layerName)) { + supportedNotOriginal.erase(layerName); + } + } + unsupportedNotOriginal.clear(); + // 1. Constants are marked as supported when all outputs can be offloaded to GPU for (const auto& op : constants) { bool is_supported = true; + for (size_t i = 0; i < op->get_output_size(); i++) { auto outTensors = op->get_output_target_inputs(i); for (auto& t : outTensors) { auto output = t.get_node(); const auto& name = output->get_friendly_name(); - if (!InferenceEngine::details::contains(supported, name)) { + if (!InferenceEngine::details::contains(supported, name) && + !InferenceEngine::details::contains(supportedNotOriginal, name)) { is_supported = false; break; } } } if (is_supported) { - supported.emplace(op->get_friendly_name()); + if (InferenceEngine::details::contains(originalOpNames, op->get_friendly_name())) + supported.emplace(op->get_friendly_name()); + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) + if (InferenceEngine::details::contains(originalOpNames, fusedLayerName)) + supported.emplace(fusedLayerName); } } diff --git a/src/plugins/intel_myriad/common/src/ngraph/query_network.cpp b/src/plugins/intel_myriad/common/src/ngraph/query_network.cpp index c2290148392..459e0270a8a 100644 --- a/src/plugins/intel_myriad/common/src/ngraph/query_network.cpp +++ b/src/plugins/intel_myriad/common/src/ngraph/query_network.cpp @@ -47,6 +47,7 @@ InferenceEngine::QueryNetworkResult getQueryNetwork(const InferenceEngine::CNNNe } }; + for (InferenceEngine::details::CNNNetworkIterator itLayer{convertedNetwork}; itLayer != InferenceEngine::details::CNNNetworkIterator(); itLayer++) { @@ -55,9 +56,25 @@ InferenceEngine::QueryNetworkResult getQueryNetwork(const InferenceEngine::CNNNe continue; } + bool isSupported = false; + bool wasNodeAlreadyChecked = false; + if (InferenceEngine::details::contains(originalOps, fusedNode->get_friendly_name())) { + isSupported = isLayerSupported(itLayer); + wasNodeAlreadyChecked = true; + if (isSupported) { + supported.emplace(fusedNode->get_friendly_name()); + } else { + unsupported.emplace(fusedNode->get_friendly_name()); + } + } + for (auto& fusedLayerName : ngraph::getFusedNamesVector(fusedNode)) { if (InferenceEngine::details::contains(originalOps, fusedLayerName)) { - if (isLayerSupported(itLayer)) { + if (!wasNodeAlreadyChecked) { + isSupported = isLayerSupported(itLayer); + wasNodeAlreadyChecked = true; + } + if (isSupported) { supported.emplace(fusedLayerName); } else { unsupported.emplace(fusedLayerName); diff --git a/src/plugins/intel_myriad/myriad_plugin/myriad_plugin.cpp b/src/plugins/intel_myriad/myriad_plugin/myriad_plugin.cpp index 9a564c619d1..b71f4472a2a 100644 --- a/src/plugins/intel_myriad/myriad_plugin/myriad_plugin.cpp +++ b/src/plugins/intel_myriad/myriad_plugin/myriad_plugin.cpp @@ -180,9 +180,16 @@ QueryNetworkResult Engine::QueryNetwork( if (auto function = supportedNetwork.getFunction()) { auto clonedNetwork = cloneNetwork(supportedNetwork); + auto clonedFunction = clonedNetwork.getFunction(); auto convertedNetwork = vpu::FrontEnd::convertNetwork(clonedNetwork); QueryNetworkResult supportedRes = getQueryNetwork(clonedNetwork, function, GetName(), supportedLayers); + auto removedNodeNames = GetRemovedNodes(function, clonedFunction); + + for (const auto& layer : removedNodeNames) { + res.supportedLayersMap.emplace(layer, GetName()); + } + for (const auto& layer : supportedRes.supportedLayersMap) { res.supportedLayersMap.insert(layer); } diff --git a/src/tests/functional/plugin/cpu/behavior/hetero_cpu_query_network.cpp b/src/tests/functional/plugin/cpu/behavior/hetero_cpu_query_network.cpp new file mode 100644 index 00000000000..323cc2d2857 --- /dev/null +++ b/src/tests/functional/plugin/cpu/behavior/hetero_cpu_query_network.cpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/plugin/hetero_query_network.hpp" + +using namespace HeteroTests; + +namespace HeteroTests { + +TEST_P(HeteroQueryNetworkTest, HeteroSinglePlugin) { + std::string deviceName = GetParam(); + RunTest(deviceName); +} + +INSTANTIATE_TEST_CASE_P( + HeteroCpu, + HeteroQueryNetworkTest, + ::testing::Values( + std::string("HETERO:CPU"))); + +} // namespace HeteroTests diff --git a/src/tests/functional/plugin/gpu/behavior/hetero_gpu_query_network.cpp b/src/tests/functional/plugin/gpu/behavior/hetero_gpu_query_network.cpp new file mode 100644 index 00000000000..a04f8b80f66 --- /dev/null +++ b/src/tests/functional/plugin/gpu/behavior/hetero_gpu_query_network.cpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/plugin/hetero_query_network.hpp" + +using namespace HeteroTests; + +namespace HeteroTests { + +TEST_P(HeteroQueryNetworkTest, HeteroSinglePlugin) { + std::string deviceName = GetParam(); + RunTest(deviceName); +} + +INSTANTIATE_TEST_CASE_P( + HeteroGpu, + HeteroQueryNetworkTest, + ::testing::Values( + std::string("HETERO:GPU"))); + +} // namespace HeteroTests diff --git a/src/tests/functional/plugin/shared/include/behavior/plugin/hetero_query_network.hpp b/src/tests/functional/plugin/shared/include/behavior/plugin/hetero_query_network.hpp new file mode 100644 index 00000000000..1ea0cefa558 --- /dev/null +++ b/src/tests/functional/plugin/shared/include/behavior/plugin/hetero_query_network.hpp @@ -0,0 +1,466 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "common_test_utils/test_common.hpp" + + +#include +#include + +using namespace InferenceEngine; + +namespace HeteroTests { + +class HeteroQueryNetworkTest : public ::testing::TestWithParam { +public: + void RunTest(std::string& deviceName) { + ASSERT_GT(deviceName.size(), 0); + + //this model is a subgraph of "ctpn" model from omz + std::string model = R"V0G0N( + + + + + + + 1 + 37 + 370 + 2 + + + + + + + + 4 + + + + + + + 1 + 37 + 370 + 2 + + + 4 + + + + + 1 + 2 + 37 + 370 + + + + + + + + 4 + + + + + + + 1 + 2 + 37 + 370 + + + 4 + + + + + 1 + 37 + 370 + 2 + + + + + + + + 1 + 2 + 37 + 370 + + + + + 4 + + + + + + + + 4 + + + + + + + + + + + + + + 4 + + + 4 + + + + + + 4 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 4 + + + 1 + + + 1 + + + 1 + + + + + + + + + + + 1 + + + + + + + + 1 + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 4 + + + 1 + + + 1 + + + 1 + + + + + + + + + + + 1 + + + + + + + + 1 + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + 1 + + + 1 + + + 1 + + + + + 4 + + + + + + + + 4 + + + + + 4 + + + + + + + + 1 + 37 + 370 + 2 + + + 4 + + + + + 1 + 37 + 37 + 20 + + + + + + + 1 + 37 + 37 + 20 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +)V0G0N"; + InferenceEngine::Core ie; + Blob::Ptr weights; + + weights = make_shared_blob(TensorDesc(Precision::U8, {160}, InferenceEngine::Layout::C)); + weights->allocate(); + + auto *dataI64 = weights->buffer().as(); + auto *dataI32 = weights->buffer().as(); + dataI64[0] = 0; + dataI64[1] = 3; + dataI64[2] = 1; + dataI64[3] = 2; + + dataI64[4] = 0; + dataI64[5] = 2; + dataI64[6] = 3; + dataI64[7] = 1; + + dataI32[16] = 0; + dataI32[17] = 2; + dataI32[18] = 3; + dataI32[19] = 1; + + dataI64[10] = 0; + + dataI64[11] = 0; + dataI64[12] = 1; + dataI64[13] = 1; + + dataI64[14] = 0; + + dataI64[15] = 1; + dataI64[16] = 2; + dataI64[17] = 1; + + dataI64[18] = 0; + + dataI32[38] = 0xFFFFFFFF; + dataI32[39] = 20; + + auto network = ie.ReadNetwork(model, weights); + + QueryNetworkResult result; + OV_ASSERT_NO_THROW(result = ie.QueryNetwork(network, deviceName)); + ASSERT_EQ(27, result.supportedLayersMap.size()); + + std::set checkNames = {"input", + "rpn_cls_prob/Transpose7580/value758213165", + "rpn_cls_prob/Transpose7580", + "rpn_cls_prob/Transpose/value756213066", + "rpn_cls_prob/Transpose", + "Shape_2", + "Shape_2/GatherNCHWtoNHWC_input_port_1/value778413036", + "Shape_2/GatherNCHWtoNHWC_input_port_2/value778613297", + "Shape_2/GatherNCHWtoNHWC", + "strided_slice_6/stack", + "strided_slice_6/stack_1", + "strided_slice_6/stack_2", + "strided_slice_6", + "Reshape_2/shape/Unsqueeze_input_port_1/value", + "Reshape_2/shape/Unsqueeze", + "strided_slice_7/stack", + "strided_slice_7/stack_1", + "strided_slice_7/stack_2", + "strided_slice_7", + "Reshape_2/shape/Unsqueeze531_input_port_1/value", + "Reshape_2/shape/Unsqueeze531", + "Reshape_2/shape/Unsqueeze533", + "Reshape_2/shape/Unsqueeze535", + "Reshape_2/shape", + "Reshape_2/Cast_1", + "Reshape_2", + "Reshape_2:0"}; + + for (auto&& name : checkNames) + EXPECT_NE(result.supportedLayersMap.find(name), result.supportedLayersMap.end()); + } +}; + +} // namespace HeteroTests