IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[HETERO]: Mask subgraphs by operations (#19834)

Browse Source 
* [HETERO]: Mask subgraphs by operations

Fix GPU tests. Fix mapping of subgraphs inputs to outputs.

Raise exception on last device

Fix dot files dumping

Small fix

* [HETERO] Fix debug dump after https://github.com/openvinotoolkit/openvino/pull/19392/

* [HETERO] Second fix for debug dump after https://github.com/openvinotoolkit/openvino/pull/19392/
This commit is contained in:
Nadezhda Ageeva authored and GitHub committed 2023-09-26 19:11:11 +04:00
1 parent 1ba8005688
commit 784cce37b3
13 files changed
+1229 -399
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/plugins/hetero/CMakeLists.txt
+2 -2
View File
@@ -8,8 +8,8 @@ endif()
set(TARGET_NAME openvino_hetero_plugin)
file(GLOB SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
file(GLOB HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp)
file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
file(GLOB_RECURSE HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp)
ov_add_plugin(NAME ${TARGET_NAME}
DEVICE_NAME "HETERO"
src/plugins/hetero/src/compiled_model.cpp
+93 -250
View File
@@ -10,6 +10,7 @@
#include "graph_debug_dump.hpp"
#include "ie_plugin_config.hpp"
#include "itt.hpp"
#include "op/device_subgraph.hpp"
#include "openvino/op/util/op_types.hpp"
#include "openvino/pass/constant_folding.hpp"
#include "openvino/pass/manager.hpp"
@@ -18,7 +19,6 @@
#include "openvino/util/common_util.hpp"
#include "plugin.hpp"
#include "properties.hpp"
#include "subgraph_collector.hpp"
#include "xml_parse_utils.h"
ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model,
@@ -38,10 +38,6 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model
}
void ov::hetero::CompiledModel::compile_model(const std::shared_ptr<ov::Model>& model) {
bool dump_dot_file = m_cfg.dump_graph;
if (std::getenv("OPENVINO_HETERO_VISUALIZE"))
dump_dot_file = true;
// Calling of ConstantFolding in HETERO plugin is required because
// in some cases topology split is happening after constant subgraph.
// It may cause replacement of Constant by Parameter in such operations
@@ -51,149 +47,28 @@ void ov::hetero::CompiledModel::compile_model(const std::shared_ptr<ov::Model>&
manager.run_passes(model);
ov::SupportedOpsMap query_model_result;
auto ordered_ops = model->get_ordered_ops();
bool all_empty = true;
bool user_set_affinities = false;
// Get user defined affinity
for (const auto& node : ordered_ops) {
for (const auto& node : model->get_ordered_ops()) {
auto& node_info = node->get_rt_info();
auto it_info = node_info.find("affinity");
if (it_info != node_info.end()) {
OPENVINO_ASSERT(it_info->second.is<std::string>(), "Unexpected type of \"affinity\" attribute");
query_model_result.emplace(node->get_friendly_name(), it_info->second.as<std::string>());
all_empty = false;
user_set_affinities = true;
}
}
if (query_model_result.empty()) {
// Restore properties in order to pass "device priorities" together
// with devices properties
auto full_properties = m_cfg.get_hetero_properties();
for (const auto& property : m_cfg.get_device_properties())
full_properties[property.first] = property.second;
query_model_result = get_hetero_plugin()->query_model(model, full_properties);
}
std::unordered_set<std::string> devices;
SubgraphCollector::AffinitiesMap affinities;
// Check that all nodes has user or plugin defined affinities
for (const auto& node : ordered_ops) {
auto it_affinity = query_model_result.find(node->get_friendly_name());
if (it_affinity != query_model_result.end()) {
affinities[node] = it_affinity->second;
devices.emplace(it_affinity->second);
} else if (all_empty) {
OPENVINO_THROW("Hetero device used default fallback policy, but some layers eg: \n(Name:",
node->get_friendly_name(),
", Type: ",
node->get_type_name(),
") were not able to be assigned on any pointed device.\n",
"It happened because these layers are not supported in plugins by default.\n",
"You need to implement custom layers to support them.");
} else {
OPENVINO_THROW("Model passed to CompiledModel has affinity assigned, but some layers eg: \n(Name:",
node->get_friendly_name(),
", Type: ",
node->get_type_name(),
") were not assigned to any device.\n",
"It might happen if you assigned layers manually and missed some layers or\n",
"if you used some automatic assigning mode which decided that these layers are not\n",
"supported by any plugin");
}
}
if (dump_dot_file) {
ov::hetero::debug::dump_affinities(model, query_model_result, devices);
}
// Init subgraph collector
SubgraphCollector subgraph_collector(model, affinities);
if (dump_dot_file) {
auto subgraph_ids = subgraph_collector.get_subgraph_ids();
std::map<std::string, SubgraphCollector::SubgraphId> map_id;
for (const auto& v : subgraph_ids) {
map_id.emplace(v.first->get_friendly_name(), v.second);
}
ov::hetero::debug::dump_subgraphs(model, query_model_result, map_id);
}
// Get subgraphs sorted topologically
auto ordered_subgraphs = subgraph_collector.get_ordered_subgraphs();
// Prepare mapping between original inputs/outputs and compiled
// submodels inputs/outputs. Example:
// original input 0 -> submodel 0 input 0,
// original input 1 -> submodel 1 input 0,
// original output 0 -> submodel 1 output 0.
//
// Mapping is required only because before compilation
// submodel may be preprocessed (if legacy API used),
// so original inputs/outputs != submodels inputs/outputs
const auto& orig_parameters = model->get_parameters();
const auto& orig_results = model->get_results();
m_inputs_to_submodels_inputs.resize(orig_parameters.size());
m_outputs_to_submodels_outputs.resize(orig_results.size());
for (size_t id = 0; id < ordered_subgraphs.size(); id++) {
for (size_t i = 0; i < ordered_subgraphs[id]._parameters.size(); i++) {
for (size_t j = 0; j < orig_parameters.size(); j++)
if (ordered_subgraphs[id]._parameters[i] == orig_parameters[j])
m_inputs_to_submodels_inputs[j] = {id, i};
}
for (size_t i = 0; i < ordered_subgraphs[id]._results.size(); i++) {
for (size_t j = 0; j < orig_results.size(); j++)
if (ordered_subgraphs[id]._results[i] == orig_results[j])
m_outputs_to_submodels_outputs[j] = {id, i};
}
}
// Prepare mapping between manually splitted inputs/outputs
// to connect tensors between compiled submodels
for (const auto& kvp : subgraph_collector.get_subgraph_parameter_to_prev_result()) {
const auto& intermed_output = kvp.second;
const auto& intermed_input = kvp.first;
for (size_t id = 0; id < ordered_subgraphs.size(); id++) {
const auto& out_it = std::find(ordered_subgraphs[id]._results.begin(),
ordered_subgraphs[id]._results.end(),
intermed_output);
if (out_it != ordered_subgraphs[id]._results.end()) {
for (size_t id2 = 0; id2 < ordered_subgraphs.size(); id2++) {
if (id2 == id)
continue;
const auto& in_it = std::find(ordered_subgraphs[id2]._parameters.begin(),
ordered_subgraphs[id2]._parameters.end(),
intermed_input);
if (in_it != ordered_subgraphs[id2]._parameters.end()) {
auto out_idx = std::distance(ordered_subgraphs[id]._results.begin(), out_it);
auto in_idx = std::distance(ordered_subgraphs[id2]._parameters.begin(), in_it);
m_submodels_input_to_prev_output[{id2, in_idx}] = {id, out_idx};
}
}
}
}
}
m_compiled_submodels.resize(ordered_subgraphs.size());
std::vector<std::shared_ptr<ov::Model>> submodels(ordered_subgraphs.size());
size_t id = 0;
for (const auto& subgraph : ordered_subgraphs) {
m_compiled_submodels[id].device = subgraph._affinity;
submodels[id] = std::make_shared<ov::Model>(subgraph._results,
subgraph._sinks,
subgraph._parameters,
m_name + '_' + std::to_string(id));
m_compiled_submodels[id].model = submodels[id];
auto meta_devices = get_hetero_plugin()->get_properties_per_device(m_compiled_submodels[id].device,
m_cfg.get_device_properties());
auto compile_device_model = [&](CompiledModelDesc& compiled_model_desc, bool add_exclusive) {
auto meta_devices =
get_hetero_plugin()->get_properties_per_device(compiled_model_desc.device, m_cfg.get_device_properties());
// disable caching for subgraphs, because the whole HETERO model is cached
auto device_config = meta_devices[m_compiled_submodels[id].device];
auto device_config = meta_devices[compiled_model_desc.device];
device_config[ov::cache_dir.name()] = "";
// set exclusive_async_requests in case when model is split
if (ordered_subgraphs.size() > 1) {
if (add_exclusive) {
auto supported_internal_properties =
get_hetero_plugin()->get_core()->get_property(m_compiled_submodels[id].device,
get_hetero_plugin()->get_core()->get_property(compiled_model_desc.device,
ov::internal::supported_properties);
if (std::find(supported_internal_properties.begin(),
supported_internal_properties.end(),
@@ -202,13 +77,60 @@ void ov::hetero::CompiledModel::compile_model(const std::shared_ptr<ov::Model>&
device_config.insert(ov::internal::exclusive_async_requests(true));
}
}
m_compiled_submodels[id].compiled_model =
get_hetero_plugin()->get_core()->compile_model(m_compiled_submodels[id].model,
m_compiled_submodels[id].device,
device_config);
++id;
}
compiled_model_desc.compiled_model = get_hetero_plugin()->get_core()->compile_model(compiled_model_desc.model,
compiled_model_desc.device,
device_config);
};
if (user_set_affinities) {
// All affinities must be defined by user
ov::hetero::SubgraphsVector ordered_subgraphs;
std::tie(ordered_subgraphs, m_mapping_info) =
get_model_subgraphs(model, query_model_result, user_set_affinities, m_cfg.dump_dot_files());
m_compiled_submodels.resize(ordered_subgraphs.size());
bool add_exclusive = ordered_subgraphs.size() > 1;
size_t id = 0;
for (const auto& subgraph : ordered_subgraphs) {
m_compiled_submodels[id].device = subgraph._affinity;
m_compiled_submodels[id].model = std::make_shared<ov::Model>(subgraph._results,
subgraph._sinks,
subgraph._parameters,
m_name + '_' + std::to_string(id));
compile_device_model(m_compiled_submodels[id], add_exclusive);
++id;
}
} else {
// Restore properties in order to pass "device priorities" together
// with devices properties
auto full_properties = m_cfg.get_hetero_properties();
for (const auto& property : m_cfg.get_device_properties())
full_properties[property.first] = property.second;
// This function modifes original model
auto cloned_model = model->clone();
std::tie(query_model_result, m_mapping_info) =
get_hetero_plugin()->query_model_update(cloned_model, full_properties, true);
ov::hetero::op::DeviceSubgraphVector ordered_subgraphs;
for (const auto& op : cloned_model->get_ordered_ops()) {
if (const auto& subgraph = ov::as_type_ptr<ov::hetero::op::DeviceSubgraph>(op)) {
ordered_subgraphs.push_back(subgraph);
} else {
OPENVINO_ASSERT(ov::op::util::is_output(op) || ov::op::util::is_parameter(op) ||
ov::op::util::is_sink(op));
}
}
m_compiled_submodels.resize(ordered_subgraphs.size());
bool add_exclusive = ordered_subgraphs.size() > 1;
size_t id = 0;
for (const auto& subgraph : ordered_subgraphs) {
m_compiled_submodels[id].device = subgraph->get_affinity();
m_compiled_submodels[id].model = subgraph->get_function();
compile_device_model(m_compiled_submodels[id], add_exclusive);
++id;
}
}
set_inputs_and_outputs();
}
@@ -284,12 +206,12 @@ ov::hetero::CompiledModel::CompiledModel(std::istream& model,
auto inputs_map_node = heteroNode.child("inputs_to_submodels_inputs");
FOREACH_CHILD(xml_node, inputs_map_node, "pair") {
m_inputs_to_submodels_inputs.emplace_back(GetUInt64Attr(xml_node, "submodel_idx"),
m_mapping_info._inputs_to_submodels_inputs.emplace_back(GetUInt64Attr(xml_node, "submodel_idx"),
GetUInt64Attr(xml_node, "node_idx"));
}
auto outputs_map_node = heteroNode.child("outputs_to_submodels_outputs");
FOREACH_CHILD(xml_node, outputs_map_node, "pair") {
m_outputs_to_submodels_outputs.emplace_back(GetUInt64Attr(xml_node, "submodel_idx"),
m_mapping_info._outputs_to_submodels_outputs.emplace_back(GetUInt64Attr(xml_node, "submodel_idx"),
GetUInt64Attr(xml_node, "node_idx"));
}
auto submodels_input_to_prev_output_node = heteroNode.child("submodels_input_to_prev_output");
@@ -298,7 +220,7 @@ ov::hetero::CompiledModel::CompiledModel(std::istream& model,
GetUInt64Attr(xml_node, "in_node_idx")};
std::pair<uint64_t, uint64_t> out_pair = {GetUInt64Attr(xml_node, "out_submodel_idx"),
GetUInt64Attr(xml_node, "out_node_idx")};
m_submodels_input_to_prev_output.emplace(in_pair, out_pair);
m_mapping_info._submodels_input_to_prev_output.emplace(in_pair, out_pair);
}
// clang-format on
set_inputs_and_outputs();
@@ -329,103 +251,8 @@ std::shared_ptr<const ov::Model> ov::hetero::CompiledModel::get_runtime_model()
for (size_t i = 0; i < m_compiled_submodels.size(); i++) {
rt_models.push_back(m_compiled_submodels.at(i).compiled_model->get_runtime_model()->clone());
}
// Results which should not be present in final graph
std::set<std::string> result_names_to_be_removed;
// Remap port indexes to names, because order of them will be modified during merge
std::map<std::pair<size_t, std::string>, std::pair<size_t, std::string>> input_to_prev_output;
for (const auto& kvp : m_submodels_input_to_prev_output) {
const auto& input_node = rt_models[kvp.first.first]->inputs()[kvp.first.second].get_node();
const auto& output_node = rt_models[kvp.second.first]->outputs()[kvp.second.second].get_node();
input_to_prev_output[{kvp.first.first, input_node->get_friendly_name()}] = {kvp.second.first,
output_node->get_friendly_name()};
result_names_to_be_removed.insert(output_node->get_friendly_name());
}
int submodel_in_index = static_cast<int>(rt_models.size()) - 1;
while (submodel_in_index >= 0 && input_to_prev_output.size() > 0) {
auto& submodel_in = rt_models[submodel_in_index];
size_t port_in_index = 0;
while (port_in_index < submodel_in->get_parameters().size()) {
auto parameter_to_replace = submodel_in->get_parameters()[port_in_index];
auto item = input_to_prev_output.find({submodel_in_index, parameter_to_replace->get_friendly_name()});
if (item == input_to_prev_output.end()) {
port_in_index++;
continue;
}
auto submodel_out_index = item->second.first;
auto submodel_out_result_name = item->second.second;
auto submodel_out = rt_models.at(submodel_out_index);
// Get all results from previous subgraph except already existed in next subgraph
std::shared_ptr<ov::op::v0::Result> result_to_replace = nullptr;
ov::ResultVector add_results;
for (auto& result : submodel_out->get_results()) {
if (result->get_friendly_name() == submodel_out_result_name) {
result_to_replace = result;
}
auto it = std::find_if(submodel_in->get_results().begin(),
submodel_in->get_results().end(),
[&](const std::shared_ptr<ov::op::v0::Result>& result_to_check) {
return result_to_check == result;
});
if (it == submodel_in->get_results().end())
add_results.push_back(result);
}
OPENVINO_ASSERT(result_to_replace != nullptr);
// Get all parameters from previous subgraph except already existed in next subgraph
ov::ParameterVector add_parameters;
for (auto& parameter : submodel_out->get_parameters()) {
auto it = std::find_if(submodel_in->get_parameters().begin(),
submodel_in->get_parameters().end(),
[&](const std::shared_ptr<ov::op::v0::Parameter>& parameter_to_check) {
return parameter_to_check == parameter;
});
if (it == submodel_in->get_parameters().end())
add_parameters.push_back(parameter);
}
// Reconnect appropariate target inputs to the new source output
auto result_source = result_to_replace->get_input_source_output(0);
auto parameter_targets = parameter_to_replace->get_output_target_inputs(0);
for (auto parameter_target : parameter_targets) {
parameter_target.replace_source_output(result_source);
}
// Update parameter and results
submodel_in->remove_parameter(parameter_to_replace);
submodel_in->add_parameters(add_parameters);
submodel_in->add_results(add_results);
// Remove processed connection
input_to_prev_output.erase(item);
// Update incoming model since it is merged
for (size_t i = 0; i < rt_models.size(); i++) {
if (rt_models[i] == submodel_out) {
rt_models[i] = submodel_in;
}
}
// Start check ports from the beginning because number of ports are modified
port_in_index = 0;
}
--submodel_in_index;
}
// Finally all subgraphs should be merged into single one
OPENVINO_ASSERT(input_to_prev_output.size() == 0);
OPENVINO_ASSERT(all_of(rt_models.begin(), rt_models.end(), [&](const std::shared_ptr<ov::Model>& rt_model) {
return rt_model == rt_models[0];
}));
auto runtime_graph = rt_models[0];
// Cleanup intermidiate results
for (size_t i = 0; i < runtime_graph->get_results().size();) {
auto& result = runtime_graph->get_results()[i];
if (result_names_to_be_removed.count(result->get_friendly_name())) {
runtime_graph->remove_result(result);
} else {
i++;
}
}
ov::hetero::merge_submodels(rt_models, m_mapping_info._submodels_input_to_prev_output);
auto& runtime_graph = rt_models[0];
OPENVINO_ASSERT(runtime_graph->inputs().size() == inputs().size());
return runtime_graph;
}
@@ -524,17 +351,33 @@ const std::vector<ov::Output<const ov::Node>>& ov::hetero::CompiledModel::output
void ov::hetero::CompiledModel::set_inputs_and_outputs() {
// Restore inputs/outputs from compiled submodels
m_compiled_inputs.reserve(m_inputs_to_submodels_inputs.size());
for (const auto& it : m_inputs_to_submodels_inputs) {
m_compiled_inputs.reserve(m_mapping_info._inputs_to_submodels_inputs.size());
for (const auto& it : m_mapping_info._inputs_to_submodels_inputs) {
const auto& submodel_idx = it.first;
OPENVINO_ASSERT(submodel_idx < m_compiled_submodels.size(),
"Model contains " + std::to_string(m_compiled_submodels.size()) +
" submodels. Index is out of range: " + std::to_string(submodel_idx));
const auto& compiled_submodel = m_compiled_submodels[submodel_idx].compiled_model;
const auto& input_idx = it.second;
m_compiled_inputs.emplace_back(m_compiled_submodels[submodel_idx].compiled_model->inputs()[input_idx]);
OPENVINO_ASSERT(input_idx < compiled_submodel->inputs().size(),
"Submodel " + std::to_string(submodel_idx) + " has " +
std::to_string(compiled_submodel->inputs().size()) +
" inputs. Index is out of range: " + std::to_string(input_idx));
m_compiled_inputs.emplace_back(compiled_submodel->inputs()[input_idx]);
}
m_compiled_outputs.reserve(m_outputs_to_submodels_outputs.size());
for (const auto& it : m_outputs_to_submodels_outputs) {
m_compiled_outputs.reserve(m_mapping_info._outputs_to_submodels_outputs.size());
for (const auto& it : m_mapping_info._outputs_to_submodels_outputs) {
const auto& submodel_idx = it.first;
OPENVINO_ASSERT(submodel_idx < m_compiled_submodels.size(),
"Model contains " + std::to_string(m_compiled_submodels.size()) +
" submodels. Index is out of range: " + std::to_string(submodel_idx));
const auto& compiled_submodel = m_compiled_submodels[submodel_idx].compiled_model;
const auto& output_idx = it.second;
m_compiled_outputs.emplace_back(m_compiled_submodels[submodel_idx].compiled_model->outputs()[output_idx]);
OPENVINO_ASSERT(output_idx < compiled_submodel->outputs().size(),
"Submodel " + std::to_string(submodel_idx) + " has " +
std::to_string(compiled_submodel->outputs().size()) +
" outputs. Index is out of range: " + std::to_string(output_idx));
m_compiled_outputs.emplace_back(compiled_submodel->outputs()[output_idx]);
}
}
@@ -546,20 +389,20 @@ void ov::hetero::CompiledModel::export_model(std::ostream& model_stream) const {
heteroNode.append_attribute("name").set_value(m_name.c_str());
auto inputs_map_node = heteroNode.append_child("inputs_to_submodels_inputs");
for (const auto& it : m_inputs_to_submodels_inputs) {
for (const auto& it : m_mapping_info._inputs_to_submodels_inputs) {
auto xml_node = inputs_map_node.append_child("pair");
xml_node.append_attribute("submodel_idx").set_value(std::to_string(it.first).c_str());
xml_node.append_attribute("node_idx").set_value(std::to_string(it.second).c_str());
}
auto outputs_map_node = heteroNode.append_child("outputs_to_submodels_outputs");
for (const auto& it : m_outputs_to_submodels_outputs) {
for (const auto& it : m_mapping_info._outputs_to_submodels_outputs) {
auto xml_node = outputs_map_node.append_child("pair");
xml_node.append_attribute("submodel_idx").set_value(std::to_string(it.first).c_str());
xml_node.append_attribute("node_idx").set_value(std::to_string(it.second).c_str());
}
auto submodels_input_to_prev_output_node = heteroNode.append_child("submodels_input_to_prev_output");
for (const auto& it : m_submodels_input_to_prev_output) {
for (const auto& it : m_mapping_info._submodels_input_to_prev_output) {
auto xml_node = submodels_input_to_prev_output_node.append_child("record");
xml_node.append_attribute("in_submodel_idx").set_value(std::to_string(it.first.first).c_str());
xml_node.append_attribute("in_node_idx").set_value(std::to_string(it.first.second).c_str());
src/plugins/hetero/src/compiled_model.hpp
+2 -5
View File
@@ -7,6 +7,7 @@
#include "config.hpp"
#include "openvino/runtime/icompiled_model.hpp"
#include "openvino/runtime/so_ptr.hpp"
#include "subgraph_collector.hpp"
namespace ov {
namespace hetero {
@@ -52,11 +53,7 @@ private:
const bool m_loaded_from_cache;
std::vector<ov::Output<const ov::Node>> m_compiled_inputs;
std::vector<ov::Output<const ov::Node>> m_compiled_outputs;
std::vector<std::pair<size_t /*submodel_idx*/, size_t /*node_idx*/>> m_inputs_to_submodels_inputs,
m_outputs_to_submodels_outputs;
std::map<std::pair<size_t /*submodel_idx*/, size_t /*node_idx*/>,
std::pair<size_t /*submodel_idx*/, size_t /*node_idx*/>>
m_submodels_input_to_prev_output;
SubgraphsMappingInfo m_mapping_info;
struct CompiledModelDesc {
std::string device;
src/plugins/hetero/src/config.cpp
+7
View File
@@ -64,4 +64,11 @@ ov::AnyMap Configuration::get_hetero_properties() const {
ov::AnyMap Configuration::get_device_properties() const {
return device_properties;
}
bool Configuration::dump_dot_files() const {
bool res = dump_graph;
if (std::getenv("OPENVINO_HETERO_VISUALIZE"))
res = true;
return res;
}
src/plugins/hetero/src/config.hpp
+5 -1
View File
@@ -31,9 +31,13 @@ struct Configuration {
ov::AnyMap get_device_properties() const;
bool dump_graph;
bool dump_dot_files() const;
std::string device_priorities;
ov::AnyMap device_properties;
private:
bool dump_graph;
};
} // namespace hetero
} // namespace ov
src/plugins/hetero/src/op/device_subgraph.cpp
+61
View File
@@ -0,0 +1,61 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "device_subgraph.hpp"
using namespace ov::op::util;
ov::hetero::op::DeviceSubgraph::DeviceSubgraph(const ov::OutputVector& args,
const std::shared_ptr<ov::Model>& body,
const std::string& affinity)
: SubGraphOp(args),
_affinity{affinity} {
SubGraphOp::set_function(body);
constructor_validate_and_infer_types();
for (size_t i = 0; i < body->get_parameters().size(); ++i)
m_input_descriptions[0].push_back(std::make_shared<InvariantInputDescription>(i, i));
for (size_t i = 0; i < body->get_output_size(); ++i)
m_output_descriptions[0].push_back(std::make_shared<BodyOutputDescription>(i, i));
}
ov::hetero::op::DeviceSubgraph::DeviceSubgraph(const ov::NodeVector& args,
const std::shared_ptr<ov::Model>& body,
const std::string& affinity)
: DeviceSubgraph(as_output_vector(args), body, affinity) {}
std::shared_ptr<ov::Node> ov::hetero::op::DeviceSubgraph::clone_with_new_inputs(const ov::OutputVector& inputs) const {
return std::make_shared<DeviceSubgraph>(inputs, body().clone(), _affinity);
}
void ov::hetero::op::DeviceSubgraph::validate_and_infer_types() {
ov::ParameterVector old_parameters;
for (auto op : body_ptr()->get_parameters()) {
old_parameters.push_back(op);
}
for (size_t i = 0; i < get_input_size(); ++i) {
body_ptr()->replace_parameter(
i,
std::make_shared<ov::op::v0::Parameter>(get_input_element_type(i), get_input_partial_shape(i)));
}
body_ptr()->validate_nodes_and_infer_types();
for (size_t i = 0; i < body_ptr()->get_parameters().size(); i++) {
body_ptr()->get_parameters()[i]->set_friendly_name(old_parameters[i]->get_friendly_name());
}
set_output_size(body_ptr()->get_output_size());
for (size_t i = 0; i < get_output_size(); ++i) {
set_output_type(i, body_ptr()->get_output_element_type(i), body_ptr()->get_output_partial_shape(i));
}
}
bool ov::hetero::op::DeviceSubgraph::visit_attributes(ov::AttributeVisitor& visitor) {
visitor.on_attribute("affinity", _affinity);
visitor.on_attribute("body", body_ptr());
visitor.on_attribute("input_descriptions", m_input_descriptions[0]);
visitor.on_attribute("output_descriptions", m_output_descriptions[0]);
return true;
}
src/plugins/hetero/src/op/device_subgraph.hpp
+62
View File
@@ -0,0 +1,62 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <memory>
#include "openvino/core/model.hpp"
#include "openvino/core/rt_info.hpp"
#include "openvino/op/op.hpp"
#include "openvino/op/util/sub_graph_base.hpp"
namespace ov {
namespace hetero {
namespace op {
/**
* @interface DeviceSubgraph
* @brief An operation that is implemented by a model
*/
class DeviceSubgraph : public ov::op::util::SubGraphOp {
public:
OPENVINO_OP("DeviceSubgraph", "hetero", ov::op::util::SubGraphOp);
DeviceSubgraph() = default;
DeviceSubgraph(const OutputVector& args, const std::shared_ptr<ov::Model>& body, const std::string& affinity);
DeviceSubgraph(const NodeVector& args, const std::shared_ptr<ov::Model>& body, const std::string& affinity);
bool visit_attributes(AttributeVisitor& visitor) override;
void validate_and_infer_types() override;
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& inputs) const override;
std::string get_affinity() {
return _affinity;
}
private:
const ov::Model& body() const {
return *m_bodies[0];
}
ov::Model& body() {
return *m_bodies[0];
}
const std::shared_ptr<ov::Model>& body_ptr() const {
return m_bodies[0];
}
std::shared_ptr<ov::Model>& body_ptr() {
return m_bodies[0];
}
std::string _affinity;
};
using DeviceSubgraphVector = std::vector<std::shared_ptr<ov::hetero::op::DeviceSubgraph>>;
} // namespace op
} // namespace hetero
} // namespace ov
src/plugins/hetero/src/plugin.cpp
+38 -19
View File
@@ -68,32 +68,51 @@ ov::hetero::Plugin::DeviceProperties ov::hetero::Plugin::get_properties_per_devi
return device_properties;
}
std::pair<ov::SupportedOpsMap, ov::hetero::SubgraphsMappingInfo> ov::hetero::Plugin::query_model_update(
std::shared_ptr<ov::Model>& model,
const ov::AnyMap& properties,
bool allow_exception) const {
Configuration full_config{properties, m_cfg};
DeviceProperties properties_per_device =
get_properties_per_device(full_config.device_priorities, full_config.get_device_properties());
// WARNING: Here is devices with user set priority
auto device_names = ov::DeviceIDParser::get_hetero_devices(full_config.device_priorities);
auto update_supported_ops = [](ov::SupportedOpsMap& final_results, const ov::SupportedOpsMap& device_results) {
for (const auto& layer_query_result : device_results)
final_results.emplace(layer_query_result);
};
ov::SupportedOpsMap supported_ops_temp;
ov::SupportedOpsMap supported_ops_final;
std::map<std::string, ov::SupportedOpsMap> query_results;
ov::hetero::SubgraphsMappingInfo mapping_info;
for (const auto& device_name : device_names) {
// If there are some unsupported operations and it is a last device
// exception should be raised when allowed
const auto& default_device = (!allow_exception || device_name != device_names.back()) ? get_device_name() : "";
const auto& device_config = properties_per_device.at(device_name);
query_results[device_name] = get_core()->query_model(model, device_name, device_config);
// Update supported operations map which includes new operations
update_supported_ops(supported_ops_temp, query_results[device_name]);
// Update supported operations map which includes original operations only
update_supported_ops(supported_ops_final, query_results[device_name]);
mapping_info =
ov::hetero::mask_model_subgraphs_by_ops(model, supported_ops_temp, m_cfg.dump_dot_files(), default_device);
}
return {supported_ops_final, mapping_info};
}
ov::SupportedOpsMap ov::hetero::Plugin::query_model(const std::shared_ptr<const ov::Model>& model,
const ov::AnyMap& properties) const {
OV_ITT_SCOPED_TASK(itt::domains::Hetero, "Plugin::query_model");
OPENVINO_ASSERT(model, "OpenVINO Model is empty!");
Configuration full_config{properties, m_cfg};
DeviceProperties properties_per_device =
get_properties_per_device(full_config.device_priorities, full_config.get_device_properties());
std::shared_ptr<ov::Model> query_model = model->clone();
std::map<std::string, ov::SupportedOpsMap> query_results;
for (const auto& it : properties_per_device) {
const auto& device_name = it.first;
const auto& device_config = it.second;
query_results[device_name] = get_core()->query_model(model, device_name, device_config);
}
// WARNING: Here is devices with user set priority
auto device_names = ov::DeviceIDParser::get_hetero_devices(full_config.device_priorities);
ov::SupportedOpsMap res;
for (const auto& device_name : device_names)
for (const auto& layer_query_result : query_results[device_name])
res.emplace(layer_query_result);
return res;
return query_model_update(query_model, properties).first;
}
void ov::hetero::Plugin::set_property(const ov::AnyMap& properties) {
src/plugins/hetero/src/plugin.hpp
+6
View File
@@ -13,6 +13,7 @@
#include "config.hpp"
#include "openvino/runtime/iplugin.hpp"
#include "subgraph_collector.hpp"
namespace ov {
namespace hetero {
@@ -59,6 +60,11 @@ private:
DeviceProperties get_properties_per_device(const std::string& device_priorities,
const ov::AnyMap& properties) const;
std::pair<ov::SupportedOpsMap, ov::hetero::SubgraphsMappingInfo> query_model_update(
std::shared_ptr<ov::Model>& model,
const ov::AnyMap& properties,
bool allow_exception = false) const;
Configuration m_cfg;
};
src/plugins/hetero/src/subgraph_collector.cpp
+533 -72
View File
@@ -6,9 +6,14 @@
#include <deque>
#include "graph_debug_dump.hpp"
#include "op/device_subgraph.hpp"
#include "openvino/core/except.hpp"
#include "openvino/core/graph_util.hpp"
#include "openvino/core/rt_info.hpp"
#include "openvino/op/util/op_types.hpp"
#include "openvino/util/common_util.hpp"
#include "transformations/utils/utils.hpp"
namespace {
@@ -32,18 +37,44 @@ static bool intersects(const Set& lhs, const Set& rhs) {
return true;
return false;
}
template <typename T>
static std::vector<T> addition(const std::vector<T>& vector1, const std::vector<T>& vector2) {
std::vector<T> addition;
std::copy_if(vector1.begin(), vector1.end(), std::back_inserter(addition), [&vector2](const T& arg) {
return (std::find(vector2.begin(), vector2.end(), arg) == vector2.end());
});
return addition;
}
template <typename T>
static size_t get_index(const std::vector<T>& vector, const T& item) {
const auto& it = std::find(vector.begin(), vector.end(), item);
OPENVINO_ASSERT(it != vector.end());
return static_cast<size_t>(std::distance(vector.begin(), it));
}
} // namespace
std::shared_ptr<ov::Node> ov::hetero::SubgraphCollector::output_node(
const ov::hetero::SubgraphCollector::Output& output) const {
return output.get_node_shared_ptr();
}
std::shared_ptr<ov::Node> ov::hetero::SubgraphCollector::input_node(
const ov::hetero::SubgraphCollector::Input& input) const {
return input.get_source_output().get_node_shared_ptr();
return output_node(input.get_source_output());
}
ov::hetero::SubgraphCollector::SubgraphCollector(const std::shared_ptr<ov::Model>& model,
const AffinitiesMap& affinities)
: _ordered_ops{model->get_ordered_ops()},
_origin_parameters(model->get_parameters()),
_origin_results(model->get_results()),
_origin_sinks(model->get_sinks()),
_intermediate_parameters{},
_intermediate_results(),
_affinities{affinities},
_graph_input_nodes{},
_node_input_dependencies{},
_subgraph_inputs{},
_subgraph_parameter_to_prev_result{} {
@@ -52,11 +83,14 @@ ov::hetero::SubgraphCollector::SubgraphCollector(const std::shared_ptr<ov::Model
_subgraph_ids = collect_subgraphs_ids();
}
bool ov::hetero::SubgraphCollector::is_graph_input_node(const ov::Node* node) const {
return ov::op::util::is_parameter(node) || ov::op::util::is_constant(node);
}
void ov::hetero::SubgraphCollector::init() {
// Get all subgraph inputs using just node affinities. Also collect transitive closure
for (const auto& node : _ordered_ops) {
if (ov::op::util::is_parameter(node) || ov::op::util::is_constant(node)) {
_graph_input_nodes.insert(node);
if (is_graph_input_node(node.get())) {
_subgraph_inputs.insert(Input{node.get(), 0});
_node_input_dependencies[node].insert(Input{node.get(), 0});
} else {
@@ -64,10 +98,14 @@ void ov::hetero::SubgraphCollector::init() {
auto& node_input_dependency = _node_input_dependencies[node];
for (const auto& input : inputs) {
node_input_dependency.insert(input);
auto& inputDependency = _node_input_dependencies[input_node(input)];
node_input_dependency.insert(inputDependency.begin(), inputDependency.end());
auto& input_dependency = _node_input_dependencies[input_node(input)];
node_input_dependency.insert(input_dependency.begin(), input_dependency.end());
if (_affinities.at(node) != _affinities.at(input_node(input))) {
_subgraph_inputs.insert(input);
if (ov::op::util::is_output(node)) {
_affinities[node] = _affinities.at(input_node(input));
} else {
_subgraph_inputs.insert(input);
}
}
}
}
@@ -81,23 +119,22 @@ void ov::hetero::SubgraphCollector::split_cyclic_dependencies() {
prev_subgraphs = _subgraph_inputs.size();
auto subgraph_ids = collect_subgraphs_ids();
// All inputs that belong to the same subgraph as node
std::unordered_map<std::shared_ptr<ov::Node>, InputSet> node_subgraph_input_dependencies;
NodeMap<InputSet> node_subgraph_input_dependencies;
// All inputs that depends on the same subgraph as node
std::unordered_map<std::shared_ptr<ov::Node>, InputSet> node_subgraph_cyclic_iput_dependencies;
NodeMap<InputSet> node_subgraph_cyclic_iput_dependencies;
for (const auto& node : _ordered_ops) {
auto& node_subgraph_input_dependency = node_subgraph_input_dependencies[node];
auto all_node_subgraph_inputs = intersection(_node_input_dependencies[node], _subgraph_inputs);
for (const auto& subgraphInput : all_node_subgraph_inputs) {
if (subgraph_ids[node] == subgraph_ids[subgraphInput.get_node()->shared_from_this()]) {
node_subgraph_input_dependency.emplace(subgraphInput);
for (const auto& subgraph_input : all_node_subgraph_inputs) {
if (subgraph_ids[node] == subgraph_ids[subgraph_input.get_node()->shared_from_this()]) {
node_subgraph_input_dependency.emplace(subgraph_input);
}
}
auto& node_subgraph_cyclic_input_dependency = node_subgraph_cyclic_iput_dependencies[node];
for (const auto& subgraphInput : all_node_subgraph_inputs) {
if (!ov::op::util::is_parameter(subgraphInput.get_node()) &&
!ov::op::util::is_constant(subgraphInput.get_node()) &&
subgraph_ids[node] == subgraph_ids[input_node(subgraphInput)]) {
node_subgraph_cyclic_input_dependency.emplace(subgraphInput);
for (const auto& subgraph_input : all_node_subgraph_inputs) {
if (!is_graph_input_node(subgraph_input.get_node()) &&
subgraph_ids[node] == subgraph_ids[input_node(subgraph_input)]) {
node_subgraph_cyclic_input_dependency.emplace(subgraph_input);
}
}
}
@@ -106,10 +143,10 @@ void ov::hetero::SubgraphCollector::split_cyclic_dependencies() {
auto& node_subgraph_cyclic_input_dependency = node_subgraph_cyclic_iput_dependencies[node];
if (!node_subgraph_cyclic_input_dependency.empty()) {
// Collect all subgraph inputs that cyclic subgraph output depends on
InputSet cyclicInputsDependencies;
InputSet cyclic_inputs_dependencies;
for (const auto& cyclicInput : node_subgraph_cyclic_input_dependency) {
for (const auto& input : node_subgraph_input_dependencies[input_node(cyclicInput)]) {
cyclicInputsDependencies.emplace(input);
cyclic_inputs_dependencies.emplace(input);
}
}
for (const auto& input : node->inputs()) {
@@ -118,7 +155,7 @@ void ov::hetero::SubgraphCollector::split_cyclic_dependencies() {
auto& input_node_subgraph_input_dependency = node_subgraph_input_dependencies[input_node(input)];
if (!intersects(node_subgraph_cyclic_input_dependency,
input_node_subgraph_cyclic_input_dependency) &&
intersects(cyclicInputsDependencies, input_node_subgraph_input_dependency)) {
intersects(cyclic_inputs_dependencies, input_node_subgraph_input_dependency)) {
_subgraph_inputs.insert(input);
}
}
@@ -132,7 +169,7 @@ ov::hetero::SubgraphCollector::SubgraphIdsMap ov::hetero::SubgraphCollector::col
NodeMap<SubgraphId*> subgraph_id_ptrs;
for (const auto& node : _ordered_ops) {
auto all_node_inputs = node->inputs();
std::vector<Input> inputs;
InputVector inputs;
for (const auto& input : all_node_inputs) {
if (_subgraph_inputs.find(input) == _subgraph_inputs.end()) {
inputs.emplace_back(std::move(input));
@@ -160,49 +197,56 @@ ov::hetero::SubgraphCollector::SubgraphIdsMap ov::hetero::SubgraphCollector::col
}
return result;
}
void ov::hetero::SubgraphCollector::split_subgraphs_by_parameter_results() {
// Break graph using insertion of result parameter split
std::vector<std::shared_ptr<ov::op::v0::Result>> results;
{
std::set<ov::Output<ov::Node>> subgraph_outputs;
for (const auto& input : _subgraph_inputs) {
if (!ov::op::util::is_parameter(input.get_node()) && !ov::op::util::is_constant(input.get_node())) {
auto input_source_output = input.get_source_output();
if (!ov::op::util::is_parameter(input_source_output.get_node()) &&
!ov::op::util::is_constant(input_source_output.get_node())) {
subgraph_outputs.insert(input_source_output);
}
// Sort subgraph inputs by order
InputVector ordered_subgraph_inputs;
for (auto& op : _ordered_ops) {
for (auto& input : op->inputs()) {
if (_subgraph_inputs.count(input)) {
ordered_subgraph_inputs.push_back(input);
}
}
for (const auto& output : subgraph_outputs) {
auto output_subgraph_id = _subgraph_ids.at(output.get_node_shared_ptr());
auto inputs = output.get_target_inputs();
// Collect input subsets from other subgraphs. Each subset of inputs belongs to the same subgraph
std::map<int, std::set<ov::Input<ov::Node>>> input_subsets;
for (const auto& input : inputs) {
auto input_subgraph_id = _subgraph_ids.at(input.get_node()->shared_from_this());
if (output_subgraph_id != input_subgraph_id) {
input_subsets[input_subgraph_id].emplace(input);
}
}
// Collect subgraph ordered subgraph outputs
OutputVector ordered_subgraph_outputs;
for (const auto& input : ordered_subgraph_inputs) {
if (!is_graph_input_node(input.get_node())) {
auto input_source_output = input.get_source_output();
if (!is_graph_input_node(input_source_output.get_node())) {
ordered_subgraph_outputs.push_back(input_source_output);
}
}
}
// Break graph using insertion of result parameter split
for (const auto& output : ordered_subgraph_outputs) {
auto output_subgraph_id = _subgraph_ids.at(output.get_node_shared_ptr());
auto inputs = output.get_target_inputs();
// Collect input subsets from other subgraphs. Each subset of inputs belongs to the same subgraph
std::map<SubgraphId, InputSet> input_subsets;
for (const auto& input : inputs) {
auto input_subgraph_id = _subgraph_ids.at(input.get_node()->shared_from_this());
if (output_subgraph_id != input_subgraph_id) {
input_subsets[input_subgraph_id].emplace(input);
}
}
if (input_subsets.size()) {
// Avoid duplicate results on the same output port
auto result = std::make_shared<ov::op::v0::Result>(output);
const auto name = output.get_node()->get_friendly_name() + "_" + std::to_string(output.get_index());
result->set_friendly_name(name + "_result");
ov::copy_runtime_info(output.get_node_shared_ptr(), result);
_subgraph_ids.emplace(result, output_subgraph_id);
results.push_back(result);
_intermediate_results.push_back(result);
for (const auto& input_subset : input_subsets) {
const auto input_subgraph_id = input_subset.first;
const auto inputs = input_subset.second;
// Avoid duplicate parameters in the same subgraph
auto parameter =
std::make_shared<ov::op::v0::Parameter>(output.get_element_type(), output.get_partial_shape());
parameter->set_friendly_name(name + "_parameter");
for (const auto& input : input_subset.second) {
_intermediate_parameters.push_back(parameter);
for (const auto& input : inputs) {
output.remove_target_input(input);
ov::copy_runtime_info(input.get_node()->shared_from_this(), parameter);
input.replace_source_output(parameter->output(0));
_subgraph_ids.emplace(parameter, input_subset.first);
_subgraph_ids.emplace(parameter, input_subgraph_id);
_subgraph_parameter_to_prev_result.emplace(parameter, result);
}
}
@@ -210,7 +254,7 @@ void ov::hetero::SubgraphCollector::split_subgraphs_by_parameter_results() {
}
}
std::vector<ov::hetero::SubgraphCollector::Subgraph> ov::hetero::SubgraphCollector::get_ordered_subgraphs() {
std::pair<ov::hetero::SubgraphsVector, ov::hetero::SubgraphsMappingInfo> ov::hetero::SubgraphCollector::run() {
// Break graph using insertion of result parameter split
split_subgraphs_by_parameter_results();
@@ -218,27 +262,27 @@ std::vector<ov::hetero::SubgraphCollector::Subgraph> ov::hetero::SubgraphCollect
auto subgraphs = collect_subgraphs();
// Subgraph topological sort
std::vector<Subgraph> all_subgraphs;
SubgraphsVector all_subgraphs;
for (const auto& subgraph : subgraphs) {
all_subgraphs.emplace_back(std::move(subgraph.second));
}
std::vector<Subgraph> ordered_subgraphs;
SubgraphsVector ordered_subgraphs;
using NodeSet = std::unordered_set<std::shared_ptr<ov::Node>>;
NodeSet prev_results;
size_t subgraph_topo_sorts_step = 0;
do {
OPENVINO_ASSERT(subgraph_topo_sorts_step < subgraphs.size(), "Cannot sort subgraphs!");
++subgraph_topo_sorts_step;
std::vector<Subgraph> new_ordered_subgraphs;
SubgraphsVector new_ordered_subgraphs;
auto is_ordered_subgraph = [&](const Subgraph& subgraph) {
auto& parameters = subgraph._parameters;
return std::all_of(
parameters.begin(),
parameters.end(),
[&](const ov::ParameterVector::value_type& parameter) {
return (_graph_input_nodes.find(parameter) != _graph_input_nodes.end()) ||
(prev_results.find(_subgraph_parameter_to_prev_result[parameter]) != prev_results.end());
});
return std::all_of(parameters.begin(),
parameters.end(),
[&](const ov::ParameterVector::value_type& parameter) {
return (ov::util::contains(_origin_parameters, parameter) ||
prev_results.count(_subgraph_parameter_to_prev_result[parameter]));
});
};
std::remove_copy_if(std::begin(all_subgraphs),
std::end(all_subgraphs),
@@ -257,27 +301,444 @@ std::vector<ov::hetero::SubgraphCollector::Subgraph> ov::hetero::SubgraphCollect
std::end(new_ordered_subgraphs),
std::back_inserter(ordered_subgraphs));
} while (!all_subgraphs.empty());
return ordered_subgraphs;
// Get submodels mapping information
auto mapping_info = get_subgraphs_mapping_info(ordered_subgraphs);
return {ordered_subgraphs, mapping_info};
}
std::unordered_map<ov::hetero::SubgraphCollector::SubgraphId, ov::hetero::SubgraphCollector::Subgraph>
std::unordered_map<ov::hetero::SubgraphCollector::SubgraphId, ov::hetero::Subgraph>
ov::hetero::SubgraphCollector::collect_subgraphs() {
std::unordered_map<SubgraphId, Subgraph> subgraphs;
// Extracts subgraph parameters, results and affinities
for (const auto& subgraph_id_ptr_value : _subgraph_ids) {
auto node = subgraph_id_ptr_value.first;
auto& subgraph = subgraphs[subgraph_id_ptr_value.second];
auto update_subgraph = [&](SubgraphId subgraph_id, const std::shared_ptr<ov::Node>& node) {
auto& subgraph = subgraphs[subgraph_id];
auto update_affinity = [&](const std::shared_ptr<ov::Node>& node) {
auto it_affinity = _affinities.find(node);
if (it_affinity != _affinities.end()) {
subgraph._affinity = it_affinity->second;
}
};
if (ov::op::util::is_output(node)) {
subgraph._results.emplace_back(ov::as_type_ptr<ov::op::v0::Result>(node->shared_from_this()));
subgraph._results.emplace_back(ov::as_type_ptr<ov::op::v0::Result>(node));
update_affinity(input_node(node->input(0)));
} else if (ov::op::util::is_parameter(node)) {
subgraph._parameters.emplace_back(ov::as_type_ptr<ov::op::v0::Parameter>(node->shared_from_this()));
subgraph._parameters.emplace_back(ov::as_type_ptr<ov::op::v0::Parameter>(node));
update_affinity(output_node(node->output(0)));
} else if (ov::op::util::is_sink(node)) {
subgraph._sinks.emplace_back(ov::as_type_ptr<ov::op::Sink>(node->shared_from_this()));
subgraph._sinks.emplace_back(ov::as_type_ptr<ov::op::Sink>(node));
update_affinity(input_node(node->input(0)));
}
auto it_affinity = _affinities.find(node);
if (it_affinity != _affinities.end()) {
subgraph._affinity = it_affinity->second;
};
// Update subgraph parameters
for (auto& op_vec : {_origin_parameters, _intermediate_parameters}) {
for (const auto& op : op_vec) {
const auto node = std::dynamic_pointer_cast<ov::Node>(op);
auto subgraph_id = _subgraph_ids.at(node);
update_subgraph(subgraph_id, node);
}
}
// Update subgraph results
for (auto& op_vec : {_origin_results, _intermediate_results}) {
for (const auto& op : op_vec) {
const auto node = std::dynamic_pointer_cast<ov::Node>(op);
auto subgraph_id = _subgraph_ids.at(node);
update_subgraph(subgraph_id, node);
}
}
// Update subgraph sinks
for (const auto& op : _origin_sinks) {
const auto node = std::dynamic_pointer_cast<ov::Node>(op);
auto subgraph_id = _subgraph_ids.at(node);
update_subgraph(subgraph_id, node);
}
return subgraphs;
}
ov::hetero::SubgraphsMappingInfo ov::hetero::SubgraphCollector::get_subgraphs_mapping_info(
const SubgraphsVector& ordered_subgraphs) {
SubgraphsMappingInfo info;
// Prepare mapping between original inputs/outputs and compiled
// submodels inputs/outputs. Example:
// original input 0 -> submodel 0 input 0,
// original input 1 -> submodel 1 input 0,
// original output 0 -> submodel 1 output 0.
//
// Mapping is required only because before compilation
// submodel may be preprocessed (if legacy API used),
// so original inputs/outputs != submodels inputs/outputs
info._inputs_to_submodels_inputs.resize(_origin_parameters.size());
info._outputs_to_submodels_outputs.resize(_origin_results.size());
for (size_t id = 0; id < ordered_subgraphs.size(); id++) {
for (size_t i = 0; i < ordered_subgraphs[id]._parameters.size(); i++) {
for (size_t j = 0; j < _origin_parameters.size(); j++)
if (ordered_subgraphs[id]._parameters[i] == _origin_parameters[j])
info._inputs_to_submodels_inputs[j] = {id, i};
}
for (size_t i = 0; i < ordered_subgraphs[id]._results.size(); i++) {
for (size_t j = 0; j < _origin_results.size(); j++)
if (ordered_subgraphs[id]._results[i] == _origin_results[j])
info._outputs_to_submodels_outputs[j] = {id, i};
}
}
// Prepare mapping between manually splitted inputs/outputs
// to connect tensors between compiled submodels
for (const auto& kvp : _subgraph_parameter_to_prev_result) {
const auto& intermed_output = ov::as_type_ptr<ov::op::v0::Result>(kvp.second);
const auto& intermed_input = ov::as_type_ptr<ov::op::v0::Parameter>(kvp.first);
for (size_t out_subgraph_index = 0; out_subgraph_index < ordered_subgraphs.size(); out_subgraph_index++) {
if (ov::util::contains(ordered_subgraphs[out_subgraph_index]._results, intermed_output)) {
for (size_t in_subgraph_index = 0; in_subgraph_index < ordered_subgraphs.size(); in_subgraph_index++) {
if (in_subgraph_index == out_subgraph_index)
continue;
if (ov::util::contains(ordered_subgraphs[in_subgraph_index]._parameters, intermed_input)) {
auto out_idx = get_index(ordered_subgraphs[out_subgraph_index]._results, intermed_output);
auto in_idx = get_index(ordered_subgraphs[in_subgraph_index]._parameters, intermed_input);
info._submodels_input_to_prev_output[{in_subgraph_index, in_idx}] = {out_subgraph_index,
out_idx};
}
}
}
}
}
return info;
}
void ov::hetero::merge_submodels(std::vector<std::shared_ptr<ov::Model>>& submodels,
const std::map<NodeInfo, NodeInfo>& submodels_input_to_prev_output) {
// Results which should not be present in final graph
std::set<std::string> result_names_to_be_removed;
// Remap port indexes to names, because order of them will be modified during merge
std::map<std::pair<size_t, std::string>, std::pair<size_t, std::string>> input_to_prev_output;
for (const auto& kvp : submodels_input_to_prev_output) {
const auto& input_node = submodels[kvp.first.first]->inputs()[kvp.first.second].get_node();
const auto& output_node = submodels[kvp.second.first]->outputs()[kvp.second.second].get_node();
input_to_prev_output[{kvp.first.first, input_node->get_friendly_name()}] = {kvp.second.first,
output_node->get_friendly_name()};
result_names_to_be_removed.insert(output_node->get_friendly_name());
}
int submodel_in_index = static_cast<int>(submodels.size()) - 1;
while (submodel_in_index >= 0 && input_to_prev_output.size() > 0) {
auto& submodel_in = submodels[submodel_in_index];
size_t port_in_index = 0;
while (port_in_index < submodel_in->get_parameters().size()) {
auto parameter_to_replace = submodel_in->get_parameters()[port_in_index];
auto item = input_to_prev_output.find({submodel_in_index, parameter_to_replace->get_friendly_name()});
if (item == input_to_prev_output.end()) {
port_in_index++;
continue;
}
auto submodel_out_index = item->second.first;
auto submodel_out_result_name = item->second.second;
auto submodel_out = submodels.at(submodel_out_index);
std::shared_ptr<ov::op::v0::Result> result_to_replace = nullptr;
for (auto& result : submodel_out->get_results()) {
if (result->get_friendly_name() == submodel_out_result_name) {
result_to_replace = result;
}
}
OPENVINO_ASSERT(result_to_replace != nullptr);
// Get all results from previous subgraph except already existed in next subgraph
auto add_results = addition(submodel_out->get_results(), submodel_in->get_results());
// Get all sinks from previous subgraph except already existed in next subgraph
auto add_sinks = addition(submodel_out->get_sinks(), submodel_in->get_sinks());
// Get all parameters from previous subgraph except already existed in next subgraph
auto add_parameters = addition(submodel_out->get_parameters(), submodel_in->get_parameters());
// Reconnect appropariate target inputs to the new source output
auto result_source = result_to_replace->get_input_source_output(0);
auto parameter_targets = parameter_to_replace->get_output_target_inputs(0);
for (auto parameter_target : parameter_targets) {
parameter_target.replace_source_output(result_source);
}
// Update parameter and results
submodel_in->remove_parameter(parameter_to_replace);
submodel_in->add_parameters(add_parameters);
submodel_in->add_results(add_results);
submodel_in->add_sinks(add_sinks);
// Remove processed connection
input_to_prev_output.erase(item);
// Update incoming model since it is merged
for (size_t i = 0; i < submodels.size(); i++) {
if (submodels[i] == submodel_out) {
submodels[i] = submodel_in;
}
}
// Start check ports from the beginning because number of ports are modified
port_in_index = 0;
}
--submodel_in_index;
}
// Finally all subgraphs should be merged into single one
OPENVINO_ASSERT(input_to_prev_output.size() == 0);
auto& result_model = submodels[0];
for (size_t i = 1; i < submodels.size(); i++) {
if (submodels[i] != result_model) {
result_model->add_parameters(submodels[i]->get_parameters());
result_model->add_results(submodels[i]->get_results());
result_model->add_sinks(submodels[i]->get_sinks());
}
submodels[i] = result_model;
}
OPENVINO_ASSERT(all_of(submodels.begin(), submodels.end(), [&](const std::shared_ptr<ov::Model>& submodel) {
return submodel == result_model;
}));
// Cleanup intermidiate results
for (size_t i = 0; i < result_model->get_results().size();) {
auto& result = result_model->get_results()[i];
if (result_names_to_be_removed.count(result->get_friendly_name())) {
result_model->remove_result(result);
} else {
i++;
}
}
submodels.resize(1);
}
std::pair<ov::hetero::SubgraphsVector, ov::hetero::SubgraphsMappingInfo> ov::hetero::get_model_subgraphs(
const std::shared_ptr<ov::Model>& model,
ov::SupportedOpsMap& supported_ops,
const bool user_set_affinities,
const bool dump_dot_files,
const std::string default_device) {
std::unordered_set<std::string> devices;
ov::hetero::SubgraphCollector::AffinitiesMap affinities;
ov::SupportedOpsMap debug_supported_ops{supported_ops};
// Check that all nodes has user or plugin defined affinitie
std::function<void(const std::shared_ptr<ov::Model>&, const std::string&)> collect_affinities =
[&](const std::shared_ptr<ov::Model>& model, const std::string& default_device) {
for (const auto& node : model->get_ordered_ops()) {
auto it_affinity = supported_ops.find(node->get_friendly_name());
if (it_affinity != supported_ops.end()) {
affinities[node] = it_affinity->second;
devices.emplace(it_affinity->second);
} else if (!default_device.empty()) {
affinities[node] = default_device;
devices.emplace(default_device);
debug_supported_ops.insert({node->get_friendly_name(), default_device});
} else if (!user_set_affinities) {
OPENVINO_THROW("Hetero device used default fallback policy, but some layers eg: \n(Name:",
node->get_friendly_name(),
", Type: ",
node->get_type_name(),
") were not able to be assigned on any pointed device.\n",
"It happened because these layers are not supported in plugins by default.\n",
"You need to implement custom layers to support them.");
} else {
OPENVINO_THROW(
"Model passed to CompiledModel has affinity assigned, but some layers eg: \n(Name:",
node->get_friendly_name(),
", Type: ",
node->get_type_name(),
") were not assigned to any device.\n",
"It might happen if you assigned layers manually and missed some layers or\n",
"if you used some automatic assigning mode which decided that these layers are not\n",
"supported by any plugin");
}
if (dump_dot_files) {
if (auto multi_subgraph_op = std::dynamic_pointer_cast<ov::op::util::MultiSubGraphOp>(node)) {
for (size_t i = 0; i < multi_subgraph_op->get_internal_subgraphs_size(); ++i) {
if (const auto& sub_graph = multi_subgraph_op->get_function(i)) {
collect_affinities(sub_graph, debug_supported_ops.at(node->get_friendly_name()));
}
}
}
}
}
};
collect_affinities(model, default_device);
if (dump_dot_files) {
ov::hetero::debug::dump_affinities(model, debug_supported_ops, devices);
}
// Init subgraph collector
ov::hetero::SubgraphCollector subgraph_collector(model, affinities);
if (dump_dot_files) {
auto subgraph_ids = subgraph_collector.get_subgraph_ids();
std::map<std::string, ov::hetero::SubgraphCollector::SubgraphId> map_id;
std::function<void(const std::shared_ptr<ov::Model>&, const ov::hetero::SubgraphCollector::SubgraphId&)>
collect_map_id = [&](const std::shared_ptr<ov::Model>& model,
const ov::hetero::SubgraphCollector::SubgraphId& default_id) {
for (const auto& node : model->get_ordered_ops()) {
ov::hetero::SubgraphCollector::SubgraphId subgraph_id;
if (subgraph_ids.count(node)) {
subgraph_id = subgraph_ids.at(node);
} else {
OPENVINO_ASSERT(default_id >= 0, "Invalid default id for node " + node->get_friendly_name());
subgraph_id = default_id;
}
map_id.emplace(node->get_friendly_name(), subgraph_id);
if (auto multi_subgraph_op = std::dynamic_pointer_cast<ov::op::util::MultiSubGraphOp>(node)) {
for (size_t i = 0; i < multi_subgraph_op->get_internal_subgraphs_size(); ++i) {
if (const auto& sub_graph = multi_subgraph_op->get_function(i)) {
collect_map_id(sub_graph, subgraph_id);
}
}
}
}
};
collect_map_id(model, -1);
ov::hetero::debug::dump_subgraphs(model, debug_supported_ops, map_id);
}
// Get subgraphs sorted topologically and appropriate mapping information
return subgraph_collector.run();
}
ov::hetero::SubgraphsMappingInfo ov::hetero::mask_model_subgraphs_by_ops(std::shared_ptr<ov::Model>& model,
ov::SupportedOpsMap& supported_ops,
const bool dump_dot_files,
const std::string default_device) {
const std::string subgraph_id_rt_info_name = "HETERO_SUBGRAPH_ID";
const std::string input_id_rt_info_name = "HETERO_INPUT_ID";
const std::string output_id_rt_info_name = "HETERO_OUTPUT_ID";
const auto name = model->get_friendly_name();
SubgraphsVector ordered_subgraphs;
SubgraphsMappingInfo mapping_info;
std::tie(ordered_subgraphs, mapping_info) =
get_model_subgraphs(model, supported_ops, false, dump_dot_files, default_device);
SubmodelsVector submodels{ordered_subgraphs.size()};
for (size_t i = 0; i < ordered_subgraphs.size(); i++) {
const auto& subgraph = ordered_subgraphs.at(i);
auto submodel_name = name + '_' + std::to_string(i);
submodels[i] =
std::make_shared<ov::Model>(subgraph._results, subgraph._sinks, subgraph._parameters, submodel_name);
const auto& submodel = submodels[i];
// Check whether model is subgraph already
bool is_subgraph = ov::op::util::has_op_with_type<ov::hetero::op::DeviceSubgraph>(submodel);
if (is_subgraph) {
for (auto& op : submodel->get_ordered_ops()) {
if (!ov::as_type_ptr<ov::hetero::op::DeviceSubgraph>(op) && !ov::op::util::is_parameter(op) &&
!ov::op::util::is_output(op) && !ov::op::util::is_sink(op)) {
is_subgraph = false;
break;
}
}
}
if (subgraph._affinity != default_device && !is_subgraph) {
// Replace submodel by subgraph operation
ParameterVector subgraph_parameters{submodel->inputs().size()};
OutputVector args{submodel->inputs().size()};
for (size_t j = 0; j < submodel->inputs().size(); j++) {
auto const& input = submodel->input(j);
subgraph_parameters[j] =
std::make_shared<ov::op::v0::Parameter>(input.get_element_type(), input.get_partial_shape());
supported_ops[subgraph_parameters[j]->get_friendly_name()] = subgraph._affinity;
args[j] = subgraph_parameters[j]->output(0);
}
auto subgraph_op = std::make_shared<ov::hetero::op::DeviceSubgraph>(args, submodel, subgraph._affinity);
supported_ops[subgraph_op->get_friendly_name()] = subgraph._affinity;
ResultVector subgraph_results{subgraph_op->outputs().size()};
for (size_t j = 0; j < subgraph_op->outputs().size(); j++) {
const auto& output = subgraph_op->output(j);
subgraph_results[j] = std::make_shared<ov::op::v0::Result>(output);
supported_ops[subgraph_results[j]->get_friendly_name()] = subgraph._affinity;
}
submodels[i] = std::make_shared<ov::Model>(subgraph_results, subgraph_parameters);
}
// Store original subgraph id
for (auto& op : submodels[i]->get_ordered_ops()) {
if (auto subgraph_op = ov::as_type_ptr<ov::hetero::op::DeviceSubgraph>(op)) {
auto& rt_info = op->get_rt_info();
rt_info[subgraph_id_rt_info_name] = i;
const auto& parameters = submodels[i]->get_parameters();
OPENVINO_ASSERT(parameters.size() == op->inputs().size());
for (auto& input : op->inputs()) {
const auto& source_output = input.get_source_output().get_node()->shared_from_this();
if (auto parameter = ov::as_type_ptr<ov::op::v0::Parameter>(source_output)) {
input.get_rt_info()[input_id_rt_info_name] = get_index(parameters, parameter);
}
}
const auto& results = submodels[i]->get_results();
OPENVINO_ASSERT(results.size() == op->outputs().size());
for (auto& output : op->outputs()) {
for (auto& input : output.get_target_inputs()) {
auto target_input = input.get_node()->shared_from_this();
if (auto result = ov::as_type_ptr<ov::op::v0::Result>(target_input)) {
output.get_rt_info()[output_id_rt_info_name] = get_index(results, result);
}
}
}
}
}
}
merge_submodels(submodels, mapping_info._submodels_input_to_prev_output);
model = submodels[0];
// Finally update mapping information according to the new operation order
std::map<size_t, size_t> subgraph_id_map;
std::map<size_t, std::map<size_t, size_t>> input_id_map;
std::map<size_t, std::map<size_t, size_t>> output_id_map;
size_t subgraph_op_id = 0;
for (auto& op : model->get_ordered_ops()) {
if (ov::as_type_ptr<ov::hetero::op::DeviceSubgraph>(op)) {
auto& rt_info = op->get_rt_info();
subgraph_id_map[rt_info[subgraph_id_rt_info_name].as<size_t>()] = subgraph_op_id;
rt_info.erase(subgraph_id_rt_info_name);
for (size_t j = 0; j < op->inputs().size(); j++) {
auto& input_rt_info = op->input(j).get_rt_info();
input_id_map[subgraph_op_id][input_rt_info[input_id_rt_info_name].as<size_t>()] = j;
input_rt_info.erase(input_id_rt_info_name);
}
for (size_t j = 0; j < op->outputs().size(); j++) {
auto& output_rt_info = op->output(j).get_rt_info();
output_id_map[subgraph_op_id][output_rt_info[output_id_rt_info_name].as<size_t>()] = j;
output_rt_info.erase(output_id_rt_info_name);
}
subgraph_op_id++;
}
}
SubgraphsMappingInfo new_mapping_info;
if (ordered_subgraphs.size() == subgraph_op_id) {
// Only if all subgraphs were replaced by subgraph operations
// we can provide appropriate mapping information
// otherwise this information is unavailable
auto get_new_subgraph_index = [&subgraph_id_map](const size_t old_subgraph_index) {
OPENVINO_ASSERT(subgraph_id_map.count(old_subgraph_index));
return subgraph_id_map.at(old_subgraph_index);
};
auto get_new_input_index = [&input_id_map](const size_t subgraph_index, const size_t old_input_index) {
OPENVINO_ASSERT(input_id_map.at(subgraph_index).count(old_input_index));
return input_id_map.at(subgraph_index).at(old_input_index);
};
auto get_new_output_index = [&output_id_map](const size_t subgraph_index, const size_t old_output_index) {
OPENVINO_ASSERT(output_id_map.at(subgraph_index).count(old_output_index));
return output_id_map.at(subgraph_index).at(old_output_index);
};
for (auto& item : mapping_info._inputs_to_submodels_inputs) {
const auto& subgraph_index = get_new_subgraph_index(item.first);
const auto& input_index = get_new_input_index(subgraph_index, item.second);
new_mapping_info._inputs_to_submodels_inputs.push_back({subgraph_index, input_index});
}
for (auto& item : mapping_info._outputs_to_submodels_outputs) {
const auto& subgraph_index = get_new_subgraph_index(item.first);
const auto& output_index = get_new_output_index(subgraph_index, item.second);
new_mapping_info._outputs_to_submodels_outputs.push_back({subgraph_index, output_index});
}
for (auto& item : mapping_info._submodels_input_to_prev_output) {
const auto& subgraph_in_index = get_new_subgraph_index(item.first.first);
const auto& input_index = get_new_input_index(subgraph_in_index, item.first.second);
const auto& subgraph_out_index = get_new_subgraph_index(item.second.first);
const auto& output_index = get_new_output_index(subgraph_out_index, item.second.second);
new_mapping_info._submodels_input_to_prev_output[{subgraph_in_index, input_index}] = {subgraph_out_index,
output_index};
}
}
return new_mapping_info;
}
src/plugins/hetero/src/subgraph_collector.hpp
+45 -13
View File
@@ -13,49 +13,81 @@
namespace ov {
namespace hetero {
using NodeInfo = std::pair<size_t /*submodel_idx*/, size_t /*node_idx*/>;
struct SubgraphsMappingInfo {
std::vector<NodeInfo> _inputs_to_submodels_inputs;
std::vector<NodeInfo> _outputs_to_submodels_outputs;
std::map<NodeInfo, NodeInfo> _submodels_input_to_prev_output;
bool empty() {
return _inputs_to_submodels_inputs.empty() && _outputs_to_submodels_outputs.empty() &&
_submodels_input_to_prev_output.empty();
}
};
struct Subgraph {
ov::ResultVector _results;
ov::ParameterVector _parameters;
ov::SinkVector _sinks;
std::string _affinity;
};
using SubgraphsVector = std::vector<Subgraph>;
using SubmodelsVector = std::vector<std::shared_ptr<ov::Model>>;
class SubgraphCollector {
public:
struct Subgraph {
ov::ResultVector _results;
ov::ParameterVector _parameters;
ov::SinkVector _sinks;
std::string _affinity;
};
template <typename T>
using NodeMap = std::unordered_map<std::shared_ptr<ov::Node>, T>;
using NodeSet = std::unordered_set<std::shared_ptr<ov::Node>>;
using AffinitiesMap = NodeMap<std::string>;
using SubgraphId = int;
using SubgraphIdsMap = NodeMap<SubgraphId>;
using ParameterResultMap = NodeMap<std::shared_ptr<ov::Node>>;
using Input = ov::Input<ov::Node>;
using Output = ov::Output<ov::Node>;
using InputSet = std::set<Input>;
using OutputSet = std::set<Output>;
using InputVector = std::vector<Input>;
using OutputVector = std::vector<Output>;
SubgraphCollector(const std::shared_ptr<ov::Model>& model, const AffinitiesMap& affinities);
SubgraphIdsMap get_subgraph_ids() {
return _subgraph_ids;
}
ParameterResultMap get_subgraph_parameter_to_prev_result() {
return _subgraph_parameter_to_prev_result;
}
std::vector<Subgraph> get_ordered_subgraphs();
std::pair<SubgraphsVector, SubgraphsMappingInfo> run();
private:
void init();
bool is_graph_input_node(const ov::Node* node) const;
void split_cyclic_dependencies();
void split_subgraphs_by_parameter_results();
SubgraphIdsMap collect_subgraphs_ids();
std::unordered_map<SubgraphId, Subgraph> collect_subgraphs();
SubgraphsMappingInfo get_subgraphs_mapping_info(const std::vector<Subgraph>& ordered_subgraphs);
std::shared_ptr<ov::Node> input_node(const Input& input) const;
std::shared_ptr<ov::Node> output_node(const Output& output) const;
ov::NodeVector _ordered_ops;
ov::ParameterVector _origin_parameters;
ov::ResultVector _origin_results;
ov::SinkVector _origin_sinks;
ov::ParameterVector _intermediate_parameters;
ov::ResultVector _intermediate_results;
AffinitiesMap _affinities;
NodeSet _graph_input_nodes;
NodeMap<InputSet> _node_input_dependencies;
InputSet _subgraph_inputs;
SubgraphIdsMap _subgraph_ids;
ParameterResultMap _subgraph_parameter_to_prev_result;
};
void merge_submodels(SubmodelsVector& submodels, const std::map<NodeInfo, NodeInfo>& submodels_input_to_prev_output);
std::pair<SubgraphsVector, SubgraphsMappingInfo> get_model_subgraphs(const std::shared_ptr<ov::Model>& model,
ov::SupportedOpsMap& supported_ops,
const bool user_set_affinities = false,
const bool dump_dot_files = false,
const std::string default_device = "");
SubgraphsMappingInfo mask_model_subgraphs_by_ops(std::shared_ptr<ov::Model>& model,
ov::SupportedOpsMap& supported_ops,
const bool dump_dot_files = false,
const std::string default_device = "");
} // namespace hetero
} // namespace ov
src/plugins/hetero/src/sync_infer_request.cpp
+3 -3
View File
@@ -24,16 +24,16 @@ ov::hetero::InferRequest::InferRequest(const std::shared_ptr<const ov::hetero::C
for (size_t i = 0; i < compiled_model->inputs().size(); i++) {
const auto& port = compiled_model->inputs()[i];
const auto& submodel_idx = compiled_model->m_inputs_to_submodels_inputs[i].first;
const auto& submodel_idx = compiled_model->m_mapping_info._inputs_to_submodels_inputs[i].first;
m_port_to_subrequest_idx[port] = submodel_idx;
}
for (size_t i = 0; i < compiled_model->outputs().size(); i++) {
const auto& port = compiled_model->outputs()[i];
const auto& submodel_idx = compiled_model->m_outputs_to_submodels_outputs[i].first;
const auto& submodel_idx = compiled_model->m_mapping_info._outputs_to_submodels_outputs[i].first;
m_port_to_subrequest_idx[port] = submodel_idx;
}
for (const auto& kvp : compiled_model->m_submodels_input_to_prev_output) {
for (const auto& kvp : compiled_model->m_mapping_info._submodels_input_to_prev_output) {
const auto& submodel_idx_in = kvp.first.first;
const auto& port_idx_in = kvp.first.second;
const auto& submodel_idx_out = kvp.second.first;
src/plugins/hetero/tests/unit/subgraph_collector.cpp
+372 -34
View File
@@ -7,6 +7,7 @@
#include <gtest/gtest.h>
#include "common_test_utils/graph_comparator.hpp"
#include "op/device_subgraph.hpp"
#include "openvino/core/except.hpp"
#include "openvino/op/ops.hpp"
@@ -30,6 +31,27 @@ std::shared_ptr<ov::Model> create_test_model() {
result->set_friendly_name("res");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
std::shared_ptr<ov::Model> create_test_model2() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1});
const_value->set_friendly_name("const_val");
auto add1 = std::make_shared<ov::op::v1::Add>(param, const_value);
add1->set_friendly_name("add1");
auto add2 = std::make_shared<ov::op::v1::Add>(add1, const_value);
add2->set_friendly_name("add2");
auto subtract = std::make_shared<ov::op::v1::Subtract>(add2, const_value);
subtract->set_friendly_name("sub");
auto add3 = std::make_shared<ov::op::v1::Add>(add1, subtract);
add3->set_friendly_name("add3");
auto reshape_val = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1});
reshape_val->set_friendly_name("reshape_val");
auto reshape = std::make_shared<ov::op::v1::Reshape>(add3, reshape_val, true);
reshape->set_friendly_name("reshape");
auto result = std::make_shared<ov::op::v0::Result>(reshape);
result->set_friendly_name("res");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
std::shared_ptr<ov::Model> create_subgraph_add() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
@@ -37,24 +59,42 @@ std::shared_ptr<ov::Model> create_subgraph_add() {
const_value->set_friendly_name("const_val");
auto add = std::make_shared<ov::op::v1::Add>(param, const_value);
add->set_friendly_name("add");
auto result = std::make_shared<ov::op::v0::Result>(add);
result->set_friendly_name("add_0_result");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
return std::make_shared<ov::Model>(ov::OutputVector{add->output(0)}, ov::ParameterVector{param});
}
std::shared_ptr<ov::Model> create_subgraph_add_add() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1});
const_value->set_friendly_name("const_val");
auto add1 = std::make_shared<ov::op::v1::Add>(param, const_value);
add1->set_friendly_name("add1");
auto add2 = std::make_shared<ov::op::v1::Add>(add1, const_value);
add2->set_friendly_name("add2");
return std::make_shared<ov::Model>(ov::OutputVector{add2->output(0), add1->output(0)}, ov::ParameterVector{param});
}
std::shared_ptr<ov::Model> create_subgraph_sub() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("add_0_parameter");
auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1});
const_value->set_friendly_name("const_val");
auto sub = std::make_shared<ov::op::v1::Subtract>(param, const_value);
sub->set_friendly_name("sub");
auto result = std::make_shared<ov::op::v0::Result>(sub);
result->set_friendly_name("sub_0_result");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
return std::make_shared<ov::Model>(ov::OutputVector{sub->output(0)}, ov::ParameterVector{param});
}
std::shared_ptr<ov::Model> create_subgraph_add_reshape() {
auto param0 = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
auto param1 = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
auto add = std::make_shared<ov::op::v1::Add>(param0, param1);
add->set_friendly_name("add_second");
auto reshape_val = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1});
reshape_val->set_friendly_name("reshape_val");
auto reshape = std::make_shared<ov::op::v1::Reshape>(add, reshape_val, true);
reshape->set_friendly_name("reshape");
auto result = std::make_shared<ov::op::v0::Result>(reshape);
result->set_friendly_name("res");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param0, param1});
}
std::shared_ptr<ov::Model> create_subgraph_reshape() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("sub_0_parameter");
auto reshape_val = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1});
reshape_val->set_friendly_name("reshape_val");
auto reshape = std::make_shared<ov::op::v1::Reshape>(param, reshape_val, true);
@@ -68,6 +108,7 @@ std::shared_ptr<ov::Model> create_subgraph_reshape() {
class SubgraphCollectorTest : public testing::Test {
void SetUp() override {
m_model = create_test_model();
m_model_ref = m_model->clone();
m_submodels = {};
m_submodels.push_back(create_subgraph_add());
m_submodels.push_back(create_subgraph_sub());
@@ -78,10 +119,11 @@ class SubgraphCollectorTest : public testing::Test {
protected:
std::shared_ptr<ov::Model> m_model;
std::shared_ptr<ov::Model> m_model_ref;
std::vector<std::shared_ptr<ov::Model>> m_submodels;
};
TEST_F(SubgraphCollectorTest, check_subgraphs) {
TEST_F(SubgraphCollectorTest, submodel_split_and_merge) {
const std::map<std::string, std::string> supported_ops = {
{"input", "MOCK.0"},
{"const_val", "MOCK.0"},
@@ -100,10 +142,13 @@ TEST_F(SubgraphCollectorTest, check_subgraphs) {
{"reshape", 3},
{"res", 3},
};
const std::map<std::string, std::string> expected_parameter_to_prev_result_names = {
{"sub_0_parameter", "sub_0_result"},
{"add_0_parameter", "add_0_result"},
};
const SubgraphsMappingInfo exptected_mapping_info = {{// inputs to submodel inputs
NodeInfo{0, 0}},
{// outputs to submodel outpts
NodeInfo{2, 0}},
{// submodel input to previous output
{NodeInfo{1, 0}, NodeInfo{0, 0}},
{NodeInfo{2, 0}, NodeInfo{1, 0}}}};
SubgraphCollector::AffinitiesMap affinities;
SubgraphCollector::SubgraphIdsMap exptected_subgraphs_ids;
const auto ordered_ops = m_model->get_ordered_ops();
@@ -114,30 +159,323 @@ TEST_F(SubgraphCollectorTest, check_subgraphs) {
OPENVINO_ASSERT(expected_ids.count(name));
exptected_subgraphs_ids[node] = expected_ids.at(name);
}
// Collect subgraphs
SubgraphCollector subgraph_collector(m_model, affinities);
auto actual_subgraphs_ids = subgraph_collector.get_subgraph_ids();
for (auto& op : ordered_ops) {
ASSERT_EQ(actual_subgraphs_ids.at(op), exptected_subgraphs_ids.at(op));
ASSERT_EQ(exptected_subgraphs_ids, actual_subgraphs_ids);
ov::hetero::SubgraphsVector actual_subgraphs;
ov::hetero::SubgraphsMappingInfo actual_mapping_info;
std::tie(actual_subgraphs, actual_mapping_info) = subgraph_collector.run();
ASSERT_EQ(3, actual_subgraphs.size());
std::vector<std::shared_ptr<ov::Model>> actual_submodels;
for (auto& actual_subgraph : actual_subgraphs) {
actual_submodels.push_back(std::make_shared<ov::Model>(actual_subgraph._results, actual_subgraph._parameters));
}
// Subgraphs are not collected yet
ASSERT_EQ(0, subgraph_collector.get_subgraph_parameter_to_prev_result().size());
// Collect subgraphs
auto actual_subgraphs = subgraph_collector.get_ordered_subgraphs();
const size_t submodels_number = 3;
ASSERT_EQ(submodels_number, actual_subgraphs.size());
auto get_submodel = [&](size_t i) {
return std::make_shared<ov::Model>(actual_subgraphs.at(i)._results, actual_subgraphs.at(i)._parameters);
};
std::pair<bool, std::string> res;
for (size_t i = 0; i < submodels_number; i++) {
auto submodel = get_submodel(i);
auto res = compare_functions(m_submodels.at(i), submodel);
for (size_t i = 0; i < actual_submodels.size(); i++) {
auto res = compare_functions(m_submodels.at(i), actual_submodels.at(i));
ASSERT_TRUE(res.first) << res.second;
}
auto actual_parameter_to_prev_result = subgraph_collector.get_subgraph_parameter_to_prev_result();
ASSERT_EQ(actual_parameter_to_prev_result.size(), expected_parameter_to_prev_result_names.size());
for (auto& item : actual_parameter_to_prev_result) {
ASSERT_EQ(item.second->get_friendly_name(),
expected_parameter_to_prev_result_names.at(item.first->get_friendly_name()));
}
// Check mapping info
ASSERT_EQ(exptected_mapping_info._inputs_to_submodels_inputs, actual_mapping_info._inputs_to_submodels_inputs);
ASSERT_EQ(exptected_mapping_info._outputs_to_submodels_outputs, actual_mapping_info._outputs_to_submodels_outputs);
ASSERT_EQ(exptected_mapping_info._submodels_input_to_prev_output,
actual_mapping_info._submodels_input_to_prev_output);
// Merge submodels into one model back
ov::hetero::merge_submodels(actual_submodels, actual_mapping_info._submodels_input_to_prev_output);
ASSERT_EQ(1, actual_submodels.size());
auto& actual_model = actual_submodels[0];
auto res = compare_functions(m_model_ref, actual_model);
ASSERT_TRUE(res.first) << res.second;
}
TEST_F(SubgraphCollectorTest, submodel_replacement_first_device) {
const std::map<std::string, std::string> supported_ops_mock0 = {
{"input", "MOCK.0"},
{"const_val", "MOCK.0"},
{"add", "MOCK.0"},
{"reshape_val", "MOCK.0"},
{"reshape", "MOCK.0"},
{"res", "MOCK.0"},
};
{
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
auto device_subgraph0 = std::make_shared<ov::hetero::op::DeviceSubgraph>(ov::OutputVector{param->output(0)},
m_submodels.at(0),
"MOCK.0");
device_subgraph0->set_friendly_name("device_subgraph0");
auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1});
const_value->set_friendly_name("const_val");
auto subtract = std::make_shared<ov::op::v1::Subtract>(device_subgraph0->output(0), const_value);
subtract->set_friendly_name("sub");
auto device_subgraph1 = std::make_shared<ov::hetero::op::DeviceSubgraph>(ov::OutputVector{subtract->output(0)},
m_submodels.at(2),
"MOCK.0");
device_subgraph1->set_friendly_name("device_subgraph1");
auto result = std::make_shared<ov::op::v0::Result>(device_subgraph1->output(0));
result->set_friendly_name("res");
m_model_ref = std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
auto supported_ops = supported_ops_mock0;
auto actual_mapping_info = ov::hetero::mask_model_subgraphs_by_ops(m_model, supported_ops, false, "TEST");
auto res = compare_functions(m_model_ref, m_model);
ASSERT_TRUE(res.first) << res.second;
// Only first device was replace, mapping is unavailable
ASSERT_TRUE(actual_mapping_info.empty());
}
TEST_F(SubgraphCollectorTest, submodel_replacement_both_devices) {
const std::map<std::string, std::string> supported_ops_mock0 = {
{"input", "MOCK.0"},
{"const_val", "MOCK.0"},
{"add", "MOCK.0"},
{"reshape_val", "MOCK.0"},
{"reshape", "MOCK.0"},
{"res", "MOCK.0"},
};
const std::map<std::string, std::string> supported_ops_mock1 = {
{"sub", "MOCK.1"},
};
const SubgraphsMappingInfo exptected_mapping_info = {{// inputs to submodel inputs
NodeInfo{0, 0}},
{// outputs to submodel outpts
NodeInfo{2, 0}},
{// submodel input to previous output
{NodeInfo{1, 0}, NodeInfo{0, 0}},
{NodeInfo{2, 0}, NodeInfo{1, 0}}}};
{
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
auto device_subgraph0 = std::make_shared<ov::hetero::op::DeviceSubgraph>(ov::OutputVector{param->output(0)},
m_submodels.at(0),
"MOCK.0");
device_subgraph0->set_friendly_name("device_subgraph0");
auto device_subgraph1 =
std::make_shared<ov::hetero::op::DeviceSubgraph>(ov::OutputVector{device_subgraph0->output(0)},
m_submodels.at(1),
"MOCK.1");
device_subgraph1->set_friendly_name("device_subgraph1");
auto device_subgraph2 =
std::make_shared<ov::hetero::op::DeviceSubgraph>(ov::OutputVector{device_subgraph1->output(0)},
m_submodels.at(2),
"MOCK.0");
device_subgraph2->set_friendly_name("device_subgraph2");
auto result = std::make_shared<ov::op::v0::Result>(device_subgraph2->output(0));
result->set_friendly_name("res");
m_model_ref = std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
ov::hetero::SubgraphsMappingInfo actual_mapping_info;
auto supported_ops = supported_ops_mock0;
ASSERT_EQ(6, supported_ops.size());
actual_mapping_info = ov::hetero::mask_model_subgraphs_by_ops(m_model, supported_ops, false, "TEST");
// + 2 subgraphs + 2 params + 2 results
ASSERT_EQ(12, supported_ops.size());
// Adds mock1 supported operations
supported_ops.insert(supported_ops_mock1.begin(), supported_ops_mock1.end());
ASSERT_EQ(13, supported_ops.size());
actual_mapping_info = ov::hetero::mask_model_subgraphs_by_ops(m_model, supported_ops, false);
// + 1 subgraphs + 1 param + 1 result
ASSERT_EQ(16, supported_ops.size());
auto res = compare_functions(m_model_ref, m_model);
ASSERT_TRUE(res.first) << res.second;
ASSERT_EQ(exptected_mapping_info._inputs_to_submodels_inputs, actual_mapping_info._inputs_to_submodels_inputs);
ASSERT_EQ(exptected_mapping_info._outputs_to_submodels_outputs, actual_mapping_info._outputs_to_submodels_outputs);
ASSERT_EQ(exptected_mapping_info._submodels_input_to_prev_output,
actual_mapping_info._submodels_input_to_prev_output);
}
class SubgraphCollectorTest2 : public SubgraphCollectorTest {
void SetUp() override {
m_model = create_test_model2();
m_model_ref = m_model->clone();
m_submodels = {};
m_submodels.push_back(create_subgraph_add_add());
m_submodels.push_back(create_subgraph_sub());
m_submodels.push_back(create_subgraph_add_reshape());
}
};
TEST_F(SubgraphCollectorTest2, submodel_split_and_merge) {
const std::map<std::string, std::string> supported_ops = {
{"input", "MOCK.0"},
{"const_val", "MOCK.0"},
{"add1", "MOCK.0"},
{"add2", "MOCK.0"},
{"sub", "MOCK.1"},
{"add3", "MOCK.0"},
{"reshape_val", "MOCK.0"},
{"reshape", "MOCK.0"},
{"res", "MOCK.0"},
};
const SubgraphsMappingInfo exptected_mapping_info = {{// inputs to submodel inputs
NodeInfo{0, 0}},
{// outputs to submodel outpts
NodeInfo{2, 0}},
{// submodel input to previous output
{NodeInfo{1, 0}, NodeInfo{0, 0}},
{NodeInfo{2, 0}, NodeInfo{0, 1}},
{NodeInfo{2, 1}, NodeInfo{1, 0}}}};
SubgraphCollector::AffinitiesMap affinities;
const auto ordered_ops = m_model->get_ordered_ops();
for (const auto& node : ordered_ops) {
const auto name = node->get_friendly_name();
OPENVINO_ASSERT(supported_ops.count(name));
affinities[node] = supported_ops.at(name);
}
// Collect subgraphs
SubgraphCollector subgraph_collector(m_model, affinities);
ov::hetero::SubgraphsVector actual_subgraphs;
ov::hetero::SubgraphsMappingInfo actual_mapping_info;
std::tie(actual_subgraphs, actual_mapping_info) = subgraph_collector.run();
ASSERT_EQ(3, actual_subgraphs.size());
std::vector<std::shared_ptr<ov::Model>> actual_submodels;
for (auto& actual_subgraph : actual_subgraphs) {
actual_submodels.push_back(std::make_shared<ov::Model>(actual_subgraph._results, actual_subgraph._parameters));
}
for (size_t i = 0; i < actual_submodels.size(); i++) {
auto res = compare_functions(m_submodels.at(i), actual_submodels.at(i));
ASSERT_TRUE(res.first) << res.second;
}
// Check mapping info
ASSERT_EQ(exptected_mapping_info._inputs_to_submodels_inputs, actual_mapping_info._inputs_to_submodels_inputs);
ASSERT_EQ(exptected_mapping_info._outputs_to_submodels_outputs, actual_mapping_info._outputs_to_submodels_outputs);
ASSERT_EQ(exptected_mapping_info._submodels_input_to_prev_output,
actual_mapping_info._submodels_input_to_prev_output);
// Merge submodels into one model back
ov::hetero::merge_submodels(actual_submodels, actual_mapping_info._submodels_input_to_prev_output);
ASSERT_EQ(1, actual_submodels.size());
auto& actual_model = actual_submodels[0];
auto res = compare_functions(m_model_ref, actual_model);
ASSERT_TRUE(res.first) << res.second;
}
TEST_F(SubgraphCollectorTest2, submodel_replacement_first_device) {
const std::map<std::string, std::string> supported_ops_mock0 = {
{"input", "MOCK.0"},
{"const_val", "MOCK.0"},
{"add1", "MOCK.0"},
{"add2", "MOCK.0"},
{"add3", "MOCK.0"},
{"reshape_val", "MOCK.0"},
{"reshape", "MOCK.0"},
{"res", "MOCK.0"},
};
{
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
auto device_subgraph0 = std::make_shared<ov::hetero::op::DeviceSubgraph>(ov::OutputVector{param->output(0)},
m_submodels.at(0),
"MOCK.0");
device_subgraph0->set_friendly_name("device_subgraph0");
auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1});
const_value->set_friendly_name("const_val");
auto subtract = std::make_shared<ov::op::v1::Subtract>(device_subgraph0->output(0), const_value);
subtract->set_friendly_name("sub");
auto device_subgraph2 = std::make_shared<ov::hetero::op::DeviceSubgraph>(
ov::OutputVector{device_subgraph0->output(0), subtract->output(0)},
m_submodels.at(2),
"MOCK.0");
device_subgraph2->set_friendly_name("device_subgraph2");
auto result = std::make_shared<ov::op::v0::Result>(device_subgraph2->output(0));
result->set_friendly_name("res");
m_model_ref = std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
auto supported_ops = supported_ops_mock0;
ASSERT_EQ(8, supported_ops.size());
auto actual_mapping_info = ov::hetero::mask_model_subgraphs_by_ops(m_model, supported_ops, false, "TEST");
// + 2 subgraphs + 3 params + 3 results
ASSERT_EQ(16, supported_ops.size());
auto res = compare_functions(m_model_ref, m_model);
ASSERT_TRUE(res.first) << res.second;
// Only first device was replace, mapping is unavailable
ASSERT_TRUE(actual_mapping_info.empty());
}
TEST_F(SubgraphCollectorTest2, submodel_replacement_both_devices) {
const std::map<std::string, std::string> supported_ops_mock0 = {
{"input", "MOCK.0"},
{"const_val", "MOCK.0"},
{"add1", "MOCK.0"},
{"add2", "MOCK.0"},
{"add3", "MOCK.0"},
{"reshape_val", "MOCK.0"},
{"reshape", "MOCK.0"},
{"res", "MOCK.0"},
};
const std::map<std::string, std::string> supported_ops_mock1 = {
{"sub", "MOCK.1"},
};
const SubgraphsMappingInfo exptected_mapping_info = {{// inputs to submodel inputs
NodeInfo{0, 0}},
{// outputs to submodel outpts
NodeInfo{2, 0}},
{// submodel input to previous output
{NodeInfo{1, 0}, NodeInfo{0, 0}},
{NodeInfo{2, 0}, NodeInfo{0, 1}},
{NodeInfo{2, 1}, NodeInfo{1, 0}}}};
{
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
auto device_subgraph0 = std::make_shared<ov::hetero::op::DeviceSubgraph>(ov::OutputVector{param->output(0)},
m_submodels.at(0),
"MOCK.0");
device_subgraph0->set_friendly_name("device_subgraph0");
auto device_subgraph1 =
std::make_shared<ov::hetero::op::DeviceSubgraph>(ov::OutputVector{device_subgraph0->output(1)},
m_submodels.at(1),
"MOCK.1");
device_subgraph1->set_friendly_name("device_subgraph1");
auto device_subgraph2 = std::make_shared<ov::hetero::op::DeviceSubgraph>(
ov::OutputVector{device_subgraph0->output(0), device_subgraph1->output(0)},
m_submodels.at(2),
"MOCK.0");
device_subgraph2->set_friendly_name("device_subgraph2");
auto result = std::make_shared<ov::op::v0::Result>(device_subgraph2->output(0));
result->set_friendly_name("res");
m_model_ref = std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
ov::hetero::SubgraphsMappingInfo actual_mapping_info;
auto supported_ops = supported_ops_mock0;
ASSERT_EQ(8, supported_ops.size());
actual_mapping_info = ov::hetero::mask_model_subgraphs_by_ops(m_model, supported_ops, false, "TEST");
// + 2 subgraphs + 3 params + 3 results
ASSERT_EQ(16, supported_ops.size());
// Adds mock1 supported operations
supported_ops.insert(supported_ops_mock1.begin(), supported_ops_mock1.end());
ASSERT_EQ(17, supported_ops.size());
actual_mapping_info = ov::hetero::mask_model_subgraphs_by_ops(m_model, supported_ops, false);
// + 1 subgraphs + 1 param + 1 result
ASSERT_EQ(20, supported_ops.size());
auto res = compare_functions(m_model_ref, m_model);
ASSERT_TRUE(res.first) << res.second;
ASSERT_EQ(exptected_mapping_info._inputs_to_submodels_inputs, actual_mapping_info._inputs_to_submodels_inputs);
ASSERT_EQ(exptected_mapping_info._outputs_to_submodels_outputs, actual_mapping_info._outputs_to_submodels_outputs);
ASSERT_EQ(exptected_mapping_info._submodels_input_to_prev_output,
actual_mapping_info._submodels_input_to_prev_output);
}