diff --git a/src/plugins/hetero/src/compiled_model.cpp b/src/plugins/hetero/src/compiled_model.cpp index 46e2419fd44..7b8bf399286 100644 --- a/src/plugins/hetero/src/compiled_model.cpp +++ b/src/plugins/hetero/src/compiled_model.cpp @@ -11,10 +11,13 @@ #include "ie_plugin_config.hpp" #include "itt.hpp" #include "openvino/op/util/op_types.hpp" +#include "openvino/pass/constant_folding.hpp" +#include "openvino/pass/manager.hpp" #include "openvino/runtime/internal_properties.hpp" #include "openvino/runtime/properties.hpp" #include "openvino/util/common_util.hpp" #include "plugin.hpp" +#include "properties.hpp" #include "xml_parse_utils.h" template @@ -55,6 +58,14 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr& model if (std::getenv("OPENVINO_HETERO_VISUALIZE")) dumpDotFile = true; + // Calling of ConstantFolding in HETERO plugin is required because + // in some cases topology split is happening after constant subgraph. + // It may cause replacement of Constant by Parameter in such operations + // like Reshape/Transpose/Gather and lead to unexpected dynamism or exception + ov::pass::Manager manager; + manager.register_pass(); + manager.run_passes(model); + ov::SupportedOpsMap queryNetworkResult; auto orderedOps = model->get_ordered_ops(); @@ -664,7 +675,8 @@ ov::Any ov::hetero::CompiledModel::get_property(const std::string& name) const { std::vector ro_properties{ov::model_name, ov::optimal_number_of_infer_requests, ov::execution_devices, - ov::loaded_from_cache}; + ov::loaded_from_cache, + ov::hetero::number_of_submodels}; return ro_properties; }; const auto& to_string_vector = [](const std::vector& properties) { @@ -723,6 +735,8 @@ ov::Any ov::hetero::CompiledModel::get_property(const std::string& name) const { device_names.push_back(comp_model_desc.device); } return decltype(ov::execution_devices)::value_type{device_names}; + } else if (ov::hetero::number_of_submodels == name) { + return decltype(ov::hetero::number_of_submodels)::value_type{m_compiled_submodels.size()}; } return m_cfg.get(name); OPENVINO_SUPPRESS_DEPRECATED_END diff --git a/src/plugins/hetero/src/properties.hpp b/src/plugins/hetero/src/properties.hpp index 799d07dbe7b..8ecd488bff9 100644 --- a/src/plugins/hetero/src/properties.hpp +++ b/src/plugins/hetero/src/properties.hpp @@ -13,5 +13,10 @@ namespace hetero { */ static constexpr Property caching_device_properties{"CACHING_DEVICE_PROPERTIES"}; +/** + * @brief Read-only property showing number of compiled submodels + */ +static constexpr Property number_of_submodels{"HETERO_NUMBER_OF_SUBMODELS"}; + } // namespace hetero } // namespace ov diff --git a/src/plugins/hetero/tests/functional/compile_model_tests.cpp b/src/plugins/hetero/tests/functional/compile_model_tests.cpp index 9fb2745f594..bfcd0ca1c8b 100644 --- a/src/plugins/hetero/tests/functional/compile_model_tests.cpp +++ b/src/plugins/hetero/tests/functional/compile_model_tests.cpp @@ -42,6 +42,20 @@ TEST_F(HeteroTests, compile_without_device_priorities_throw) { EXPECT_THROW(core.compile_model(model, "HETERO"), ov::Exception); } +TEST_F(HeteroTests, compile_dynamic_model_fail) { + // Change device priority + core.set_property("HETERO", ov::device::priorities("MOCK0,MOCK1")); + auto model = create_model_with_subtract_reshape(true); + EXPECT_THROW(core.compile_model(model, "HETERO"), ov::Exception); +} + +TEST_F(HeteroTests, compile_model_shapeof) { + // Change device priority + core.set_property("HETERO", ov::device::priorities("MOCK0,MOCK1")); + auto model = create_model_with_subtract_shapeof_reshape(); + EXPECT_NO_THROW(core.compile_model(model, "HETERO")); +} + TEST_F(HeteroTests, compile_with_device_properties) { ov::AnyMap config = {ov::device::priorities("MOCK0,MOCK1"), ov::device::properties("MOCK0", ov::num_streams(4), ov::enable_profiling(false)), diff --git a/src/plugins/hetero/tests/functional/hetero_tests.cpp b/src/plugins/hetero/tests/functional/hetero_tests.cpp index f8d1f5ea91c..4228a5c14ce 100644 --- a/src/plugins/hetero/tests/functional/hetero_tests.cpp +++ b/src/plugins/hetero/tests/functional/hetero_tests.cpp @@ -12,6 +12,8 @@ #include "openvino/core/any.hpp" #include "openvino/core/except.hpp" #include "openvino/opsets/opset11.hpp" +#include "openvino/pass/constant_folding.hpp" +#include "openvino/pass/manager.hpp" #include "openvino/pass/serialize.hpp" #include "openvino/runtime/exec_model_info.hpp" #include "openvino/runtime/internal_properties.hpp" @@ -22,6 +24,7 @@ #include "openvino/runtime/properties.hpp" #include "openvino/util/file_util.hpp" #include "openvino/util/shared_object.hpp" +#include "transformations/init_node_info.hpp" #include "transformations/rt_info/fused_names_attribute.hpp" namespace { @@ -67,8 +70,9 @@ ov::Tensor ov::hetero::tests::HeteroTests::create_and_fill_tensor(const ov::elem OPENVINO_THROW("Cannot generate tensor. Unsupported element type."); } -std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_subtract() { - auto param = std::make_shared(ov::element::i64, ov::Shape{1, 3, 2, 2}); +std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_subtract(bool dynamic) { + int64_t bs = dynamic ? -1 : 1; + auto param = std::make_shared(ov::element::i64, ov::PartialShape{bs, 3, 2, 2}); param->set_friendly_name("input"); auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); const_value->set_friendly_name("const_val"); @@ -81,8 +85,9 @@ std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_sub return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); } -std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_subtract_reshape() { - auto param = std::make_shared(ov::element::i64, ov::Shape{1, 3, 2, 2}); +std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_subtract_reshape(bool dynamic) { + int64_t bs = dynamic ? -1 : 1; + auto param = std::make_shared(ov::element::i64, ov::PartialShape{bs, 3, 2, 2}); param->set_friendly_name("input"); auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); const_value->set_friendly_name("const_val"); @@ -99,8 +104,9 @@ std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_sub return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); } -std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_subtract_reshape_relu() { - auto param = std::make_shared(ov::element::i64, ov::Shape{1, 3, 2, 2}); +std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_subtract_reshape_relu(bool dynamic) { + int64_t bs = dynamic ? -1 : 1; + auto param = std::make_shared(ov::element::i64, ov::PartialShape{bs, 3, 2, 2}); param->set_friendly_name("input"); auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); const_value->set_friendly_name("const_val"); @@ -119,8 +125,9 @@ std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_sub return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); } -std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_reshape() { - auto param = std::make_shared(ov::element::i64, ov::Shape{1, 3, 2, 2}); +std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_reshape(bool dynamic) { + int64_t bs = dynamic ? -1 : 1; + auto param = std::make_shared(ov::element::i64, ov::PartialShape{bs, 3, 2, 2}); param->set_friendly_name("input"); auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); const_value->set_friendly_name("const_val"); @@ -135,6 +142,27 @@ std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_res return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); } +std::shared_ptr ov::hetero::tests::HeteroTests::create_model_with_subtract_shapeof_reshape(bool dynamic) { + int64_t bs = dynamic ? -1 : 1; + auto param = std::make_shared(ov::element::i64, ov::PartialShape{bs, 3, 2, 2}); + param->set_friendly_name("input"); + auto reshape_val0 = ov::opset11::Constant::create(ov::element::i64, ov::Shape{2}, {bs, 12}); + reshape_val0->set_friendly_name("reshape_val0"); + auto reshape0 = std::make_shared(param, reshape_val0, true); + reshape0->set_friendly_name("reshape0"); + auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1}, {1}); + const_value->set_friendly_name("const_val"); + auto subtract = std::make_shared(reshape0, const_value); + subtract->set_friendly_name("sub"); + auto shape_of = std::make_shared(param); + shape_of->set_friendly_name("shape_of"); + auto reshape1 = std::make_shared(subtract, shape_of, true); + reshape1->set_friendly_name("reshape1"); + auto result = std::make_shared(reshape1); + result->set_friendly_name("res"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} + // Mock plugins class MockCompiledModel : public ov::ICompiledModel { @@ -386,8 +414,11 @@ public: class MockPluginBase : public ov::IPlugin { public: - MockPluginBase(const std::string& name, const std::unordered_set& supported_ops) - : m_supported_ops(supported_ops) { + MockPluginBase(const std::string& name, + const std::unordered_set& supported_ops, + bool dynamism_supported = false) + : m_supported_ops(supported_ops), + m_dynamism_supported(dynamism_supported) { set_device_name(name); } @@ -501,10 +532,24 @@ public: auto device_id = properties.count(ov::device::id.name()) ? properties.at(ov::device::id.name()).as() : m_default_device_id; - for (const auto& op : model->get_ordered_ops()) { - if (m_supported_ops.find(op->get_type_info().name) == m_supported_ops.end()) - continue; - res[op->get_friendly_name()] = get_device_name() + "." + device_id; + + auto supported = ov::get_supported_nodes( + model, + [&](std::shared_ptr& model) { + ov::pass::Manager manager; + manager.register_pass(); + manager.register_pass(); + manager.run_passes(model); + }, + [&](const std::shared_ptr& op) { + if (op->is_dynamic() && !m_dynamism_supported) + return false; + if (m_supported_ops.find(op->get_type_info().name) == m_supported_ops.end()) + return false; + return true; + }); + for (auto&& op_name : supported) { + res.emplace(op_name, get_device_name() + "." + device_id); } return res; } @@ -512,6 +557,7 @@ public: protected: std::string m_default_device_id = "0"; std::unordered_set m_supported_ops; + bool m_dynamism_supported = false; bool m_profiling = false; bool m_loaded_from_cache{false}; }; @@ -519,7 +565,7 @@ protected: class MockPluginReshape : public MockPluginBase { public: MockPluginReshape(const std::string& name) - : MockPluginBase(name, {"Parameter", "Result", "Add", "Constant", "Reshape"}) {} + : MockPluginBase(name, {"Parameter", "Result", "Add", "Constant", "Reshape"}, true) {} const ov::Version& get_const_version() override { static const ov::Version version = {CI_BUILD_NUMBER, "openvino_mock_reshape_plugin"}; diff --git a/src/plugins/hetero/tests/functional/hetero_tests.hpp b/src/plugins/hetero/tests/functional/hetero_tests.hpp index b2af29f1947..7b6e5f85fad 100644 --- a/src/plugins/hetero/tests/functional/hetero_tests.hpp +++ b/src/plugins/hetero/tests/functional/hetero_tests.hpp @@ -20,10 +20,11 @@ public: void SetUp() override; - std::shared_ptr create_model_with_subtract(); - std::shared_ptr create_model_with_subtract_reshape(); - std::shared_ptr create_model_with_subtract_reshape_relu(); - std::shared_ptr create_model_with_reshape(); + std::shared_ptr create_model_with_subtract(bool dynamic = false); + std::shared_ptr create_model_with_subtract_reshape(bool dynamic = false); + std::shared_ptr create_model_with_subtract_reshape_relu(bool dynamic = false); + std::shared_ptr create_model_with_reshape(bool dynamic = false); + std::shared_ptr create_model_with_subtract_shapeof_reshape(bool dynamic = false); ov::Tensor create_and_fill_tensor(const ov::element::Type& type, const ov::Shape& shape); private: diff --git a/src/plugins/hetero/tests/functional/query_model_tests.cpp b/src/plugins/hetero/tests/functional/query_model_tests.cpp index 9df8c2b110a..4563cdd726b 100644 --- a/src/plugins/hetero/tests/functional/query_model_tests.cpp +++ b/src/plugins/hetero/tests/functional/query_model_tests.cpp @@ -37,8 +37,14 @@ TEST_F(HeteroTests, query_model_on_mock1) { EXPECT_EQ(op.second, dev_name); names.erase(op.first); } - EXPECT_EQ(1, names.size()); - EXPECT_EQ("reshape", *names.begin()); + const std::vector unmarked_names = {"reshape_val", "reshape", "res"}; + EXPECT_EQ(unmarked_names.size(), names.size()); + for (auto& name : unmarked_names) { + auto it = names.find(name); + if (it != names.end()) + names.erase(it); + } + EXPECT_EQ(0, names.size()); } TEST_F(HeteroTests, query_model_on_mixed) { @@ -65,3 +71,30 @@ TEST_F(HeteroTests, query_model_on_mixed) { } EXPECT_EQ(0, names.size()); } + +TEST_F(HeteroTests, query_dynamic_model_on_mixed) { + const std::string dev_name0 = "MOCK0.3"; + const std::string dev_name1 = "MOCK1.2"; + ov::AnyMap config = {ov::device::priorities(dev_name0 + "," + dev_name1)}; + const auto model = create_model_with_subtract_reshape(true); + std::set supported_ops_mock0; + for (auto& op : core.query_model(model, dev_name0)) { + if (op.second == dev_name0) + supported_ops_mock0.insert(op.first); + } + const auto supported_ops = core.query_model(model, "HETERO", config); + std::unordered_set names; + for (const auto& op : model->get_ops()) { + names.insert(op->get_friendly_name()); + } + for (const auto& op : supported_ops) { + if (supported_ops_mock0.count(op.first)) + EXPECT_EQ(op.second, dev_name0); + else + EXPECT_EQ(op.second, dev_name1); + names.erase(op.first); + } + EXPECT_EQ(1, names.size()); + // fallback plugin doesn't support dynamism + ASSERT_TRUE(names.count("sub")); +}