IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[HETERO] Hetero refactor subgraph collector. Adds unit tests. (#19656)

Browse Source 
* [HETERO] Refactor subgraph collector. Add unit tests.

* [HETERO] Adds ov_hetero_unit_tests to azure

* [HETERO] Adds ov_hetero_unit_tests to github workflows

* Small updates

* Set CI_BUILD_NUMBER

* Fix cmake

* Fix cpplint

* STATIC -> OBJECT

* Fix .github/workflows/linux.yml

* Fix cmake

* Fix .github/workflows/linux_debian.yml

* Fix win build: separate version file
This commit is contained in:
Nadezhda Ageeva 2023-09-13 16:17:32 +04:00 committed by GitHub
parent 972bb73298
commit 3454139931
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 705 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.ci/azure/linux.yml
View File

@ -368,6 +368,9 @@ jobs:
- script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVProxyTests.xml
displayName: 'OV Proxy Plugin Tests'
- script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroUnitTests.xml
displayName: 'OV Hetero Unit Tests'
- script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml
displayName: 'OV Hetero Func Tests'
@ -452,7 +455,7 @@ jobs:
--junitxml=$(INSTALL_TEST_DIR)/TEST-Pyngraph.xml \
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_utils/test_utils.py
displayName: 'Python API 2.0 Tests'
# Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
- script: |
python3 -m pytest -sv $(REPO_DIR)/src/frontends/onnx/tests $(PYTHON_STATIC_ARGS) \

8
.ci/azure/linux_debian.yml
View File

@ -284,6 +284,12 @@ jobs:
LD_LIBRARY_PATH: $(INSTALL_TEST_DIR)
displayName: 'OV Proxy Tests'
- script: |
$(INSTALL_TEST_DIR)/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroUnitTests.xml
env:
LD_LIBRARY_PATH: $(INSTALL_TEST_DIR)
displayName: 'OV Hetero Unit Tests'
- script: |
$(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml
env:
@ -373,7 +379,7 @@ jobs:
env:
LD_LIBRARY_PATH: $(INSTALL_TEST_DIR)
PYTHONPATH: $(INSTALL_TEST_DIR)
displayName: 'ONNX Frontend Python Tests'
displayName: 'ONNX Frontend Python Tests'
- script: |
set -e

4
.ci/azure/mac.yml
View File

@ -189,6 +189,10 @@ jobs:
displayName: 'OV Proxy Plugin Tests'
enabled: 'false'
- script: $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroUnitTests.xml
displayName: 'OV Hetero Unit Tests'
enabled: 'false'
- script: $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml
displayName: 'OV Hetero Func Tests'
enabled: 'false'

3
.ci/azure/windows.yml
View File

@ -264,6 +264,9 @@ jobs:
- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVProxyTests.xml
displayName: 'OV Proxy Plugin Tests'
- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVHeteroUnitTests.xml
displayName: 'OV Hetero Unit Tests'
- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVHeteroFuncTests.xml
displayName: 'OV Hetero Func Tests'

5
.github/workflows/linux.yml vendored
View File

@ -460,6 +460,11 @@ jobs:
source ${{ env.INSTALL_DIR }}/setupvars.sh
${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml
- name: Hetero Unit Tests
run: |
source ${{ env.INSTALL_DIR }}/setupvars.sh
${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml
- name: Hetero Func Tests
run: |
source ${{ env.INSTALL_DIR }}/setupvars.sh

37
.github/workflows/linux_debian.yml vendored
View File

@ -86,39 +86,39 @@ jobs:
run: |
sudo -E apt update
sudo -E ${{ env.OPENVINO_REPO }}/install_build_dependencies.sh
# 'clang' is used as a default compiler
sudo apt --assume-yes install clang
sudo apt --assume-yes install --no-install-recommends libopencv-imgproc-dev libopencv-imgcodecs-dev
# Speed up build
sudo apt -y --no-install-recommends install unzip
wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip
unzip ninja-linux.zip
sudo cp -v ninja /usr/local/bin/
# Speed up tests
git clone https://github.com/google/gtest-parallel.git
- name: Install python dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/wheel/requirements-dev.txt
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements.txt
# For running Python API tests
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt
# For running Paddle frontend unit tests
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt
# For running ONNX frontend unit tests
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests/requirements.txt
# For running TensorFlow frontend unit tests
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/tensorflow/tests/requirements.txt
# For MO unit tests
python3 -m pip install -U pip
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_mxnet.txt
@ -128,7 +128,7 @@ jobs:
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_tf2.txt
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt
# for Python API tests
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements_test.txt
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements.txt
@ -260,6 +260,11 @@ jobs:
export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH
${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml
- name: Hetero Unit Tests
run: |
export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH
${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml
- name: Hetero Func Tests
run: |
export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH
@ -344,7 +349,7 @@ jobs:
run: |
# For python imports to import pybind_mock_frontend
export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:${{ env.OPENVINO_REPO }}/tools/mo:$PYTHONPATH
export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH
python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino \
@ -355,7 +360,7 @@ jobs:
run: |
# For python imports to import pybind_mock_frontend
export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:${{ env.OPENVINO_REPO }}/tools/mo:$PYTHONPATH
export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH
python3 -m pytest -sv ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests \
@ -392,7 +397,7 @@ jobs:
- name: Samples Smoke Tests
run: |
python3 -m pip install --ignore-installed PyYAML -r ${{ env.INSTALL_TEST_DIR }}/smoke_tests/requirements.txt
export LD_LIBRARY_PATH=${{ env.IE_APP_PATH }}:$LD_LIBRARY_PATH
python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/smoke_tests -k "not GNA" \
@ -409,14 +414,14 @@ jobs:
run: |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt
export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_Roll.xml
- name: TensorFlow Lite Layer Tests - TFL FE
run: |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt
export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH
# Need to be reinstalled to have correct numpy version
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_caffe.txt
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_kaldi.txt
@ -424,7 +429,7 @@ jobs:
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_tf2.txt
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_mxnet.txt
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_lite_tests/ --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tfl_fe.xml
env:
TEST_DEVICE: CPU

5
.github/workflows/windows.yml vendored
View File

@ -645,6 +645,11 @@ jobs:
run: |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml
- name: Hetero Unit Tests
shell: cmd
run: |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml
- name: Hetero Func Tests
shell: cmd
run: |

36
src/plugins/hetero/CMakeLists.txt
View File

@ -6,7 +6,7 @@ if (NOT ENABLE_HETERO)
return()
endif()
set (TARGET_NAME "openvino_hetero_plugin")
set(TARGET_NAME openvino_hetero_plugin)
file(GLOB SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
file(GLOB HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp)
@ -15,7 +15,7 @@ ov_add_plugin(NAME ${TARGET_NAME}
DEVICE_NAME "HETERO"
PSEUDO_DEVICE
SOURCES ${SOURCES} ${HEADERS}
VERSION_DEFINES_FOR src/plugin.cpp
VERSION_DEFINES_FOR src/version.cpp
ADD_CLANG_FORMAT)
ie_faster_build(${TARGET_NAME}
@ -29,6 +29,38 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME})
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
if(BUILD_SHARED_LIBS)
set(OBJ_NAME ${TARGET_NAME}_obj)
add_library(${OBJ_NAME} OBJECT ${SOURCES} ${HEADERS})
link_system_libraries(${OBJ_NAME} PUBLIC openvino::pugixml)
ov_add_version_defines(src/version.cpp ${OBJ_NAME})
target_include_directories(${OBJ_NAME}
PRIVATE
$<TARGET_PROPERTY:openvino::runtime::dev,INTERFACE_INCLUDE_DIRECTORIES>
$<TARGET_PROPERTY:openvino::itt,INTERFACE_INCLUDE_DIRECTORIES>
PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/src
$<TARGET_PROPERTY:openvino::conditional_compilation,INTERFACE_INCLUDE_DIRECTORIES>)
set_ie_threading_interface_for(${OBJ_NAME})
target_compile_definitions(${OBJ_NAME}
PRIVATE
USE_STATIC_IE IMPLEMENT_INFERENCE_ENGINE_PLUGIN IMPLEMENT_INFERENCE_EXTENSION_API
$<TARGET_PROPERTY:ngraph,INTERFACE_COMPILE_DEFINITIONS>
$<TARGET_PROPERTY:inference_engine_plugin_api,INTERFACE_COMPILE_DEFINITIONS>)
set_target_properties(${TARGET_NAME}_obj PROPERTIES EXCLUDE_FROM_ALL ON)
set_target_properties(${TARGET_NAME}_obj PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
endif()
if(ENABLE_TESTS)
add_subdirectory(tests/unit)
endif()
if(ENABLE_FUNCTIONAL_TESTS)
add_subdirectory(tests/functional)
endif()

404
src/plugins/hetero/src/compiled_model.cpp
View File

@ -18,35 +18,9 @@
#include "openvino/util/common_util.hpp"
#include "plugin.hpp"
#include "properties.hpp"
#include "subgraph_collector.hpp"
#include "xml_parse_utils.h"
template <typename T>
using NodeMap = std::unordered_map<std::shared_ptr<ov::Node>, T>;
namespace {
template <typename Set>
static Set intersection(const Set& lhs, const Set& rhs) {
Set result;
const auto& minSizeSet = (lhs.size() < rhs.size()) ? lhs : rhs;
const auto& maxSizeSet = (lhs.size() >= rhs.size()) ? lhs : rhs;
for (auto&& val : minSizeSet)
if (maxSizeSet.find(val) != maxSizeSet.end())
result.insert(val);
return result;
}
template <typename Set>
static bool intersects(const Set& lhs, const Set& rhs) {
const auto& minSizeSet = (lhs.size() < rhs.size()) ? lhs : rhs;
const auto& maxSizeSet = (lhs.size() >= rhs.size()) ? lhs : rhs;
for (auto&& val : minSizeSet)
if (maxSizeSet.find(val) != maxSizeSet.end())
return true;
return false;
}
} // namespace
ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model,
const std::shared_ptr<const ov::IPlugin>& plugin,
const Configuration& cfg)
@ -54,9 +28,19 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model
m_cfg(cfg),
m_name(model->get_friendly_name()),
m_loaded_from_cache(false) {
bool dumpDotFile = m_cfg.dump_graph;
try {
compile_model(model);
} catch (const std::exception& e) {
OPENVINO_THROW("Standard exception from compilation library: ", e.what());
} catch (...) {
OPENVINO_THROW("Generic exception is thrown");
}
}
void ov::hetero::CompiledModel::compile_model(const std::shared_ptr<ov::Model>& model) {
bool dump_dot_file = m_cfg.dump_graph;
if (std::getenv("OPENVINO_HETERO_VISUALIZE"))
dumpDotFile = true;
dump_dot_file = true;
// Calling of ConstantFolding in HETERO plugin is required because
// in some cases topology split is happening after constant subgraph.
@ -66,47 +50,39 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model
manager.register_pass<ov::pass::ConstantFolding>();
manager.run_passes(model);
ov::SupportedOpsMap queryNetworkResult;
auto orderedOps = model->get_ordered_ops();
ov::SupportedOpsMap query_model_result;
auto ordered_ops = model->get_ordered_ops();
bool allEmpty = true;
bool all_empty = true;
// Get user defined affinity
for (const auto& node : orderedOps) {
auto& nodeInfo = node->get_rt_info();
auto itInfo = nodeInfo.find("affinity");
if (itInfo != nodeInfo.end()) {
OPENVINO_ASSERT(itInfo->second.is<std::string>(), "Unexpected type of \"affinity\" attribute");
queryNetworkResult.emplace(node->get_friendly_name(), itInfo->second.as<std::string>());
allEmpty = false;
for (const auto& node : ordered_ops) {
auto& node_info = node->get_rt_info();
auto it_info = node_info.find("affinity");
if (it_info != node_info.end()) {
OPENVINO_ASSERT(it_info->second.is<std::string>(), "Unexpected type of \"affinity\" attribute");
query_model_result.emplace(node->get_friendly_name(), it_info->second.as<std::string>());
all_empty = false;
}
}
if (queryNetworkResult.empty()) {
if (query_model_result.empty()) {
// Restore properties in order to pass "device priorities" together
// with devices properties
auto full_properties = m_cfg.get_hetero_properties();
for (const auto& property : m_cfg.get_device_properties())
full_properties[property.first] = property.second;
queryNetworkResult = plugin->query_model(model, full_properties);
query_model_result = get_hetero_plugin()->query_model(model, full_properties);
}
using Input = ov::Input<ov::Node>;
using NodeSet = std::unordered_set<std::shared_ptr<ov::Node>>;
using InputSet = std::set<Input>;
auto InputNode = [](const Input& input) {
return input.get_source_output().get_node_shared_ptr();
};
std::unordered_set<std::string> devices;
NodeMap<std::string> affinities;
SubgraphCollector::AffinitiesMap affinities;
// Check that all nodes has user or plugin defined affinities
for (const auto& node : orderedOps) {
auto itAffinity = queryNetworkResult.find(node->get_friendly_name());
if (itAffinity != queryNetworkResult.end()) {
affinities[node] = itAffinity->second;
devices.emplace(itAffinity->second);
} else if (allEmpty) {
for (const auto& node : ordered_ops) {
auto it_affinity = query_model_result.find(node->get_friendly_name());
if (it_affinity != query_model_result.end()) {
affinities[node] = it_affinity->second;
devices.emplace(it_affinity->second);
} else if (all_empty) {
OPENVINO_THROW("Hetero device used default fallback policy, but some layers eg: \n(Name:",
node->get_friendly_name(),
", Type: ",
@ -126,233 +102,24 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model
}
}
if (dumpDotFile) {
ov::hetero::debug::dump_affinities(model, queryNetworkResult, devices);
if (dump_dot_file) {
ov::hetero::debug::dump_affinities(model, query_model_result, devices);
}
NodeMap<InputSet> nodeInputDependencies;
NodeSet graphInputNodes;
InputSet subgraphInputs;
// Get all subgraph inputs using just node affinities. Also collect transitive closure
for (const auto& node : orderedOps) {
if (ov::op::util::is_parameter(node) || ov::op::util::is_constant(node)) {
graphInputNodes.insert(node);
subgraphInputs.insert(Input{node.get(), 0});
nodeInputDependencies[node].insert(Input{node.get(), 0});
} else {
auto inputs = node->inputs();
auto& nodeInputDependency = nodeInputDependencies[node];
for (const auto& input : inputs) {
nodeInputDependency.insert(input);
auto& inputDependency = nodeInputDependencies[InputNode(input)];
nodeInputDependency.insert(inputDependency.begin(), inputDependency.end());
if (affinities[node] != affinities[InputNode(input)]) {
subgraphInputs.insert(input);
}
}
}
}
// Init subgraph collector
SubgraphCollector subgraph_collector(model, affinities);
// Assign each node subgraph ID
auto CollectSubgraphs = [&] {
std::deque<int> subgraphIds;
NodeMap<int*> subgraphIdPtrs;
for (const auto& node : orderedOps) {
auto allNodeInputs = node->inputs();
std::vector<Input> inputs;
for (const auto& input : allNodeInputs) {
if (subgraphInputs.find(input) == subgraphInputs.end()) {
inputs.emplace_back(std::move(input));
}
}
if (inputs.empty()) {
subgraphIds.push_back(static_cast<int>(subgraphIds.size()));
subgraphIdPtrs.emplace(node, &(subgraphIds.back()));
} else {
auto firstInputSubgraphIdPtr = subgraphIdPtrs[InputNode(inputs.front())];
for (const auto& input : inputs) {
auto inputId = *subgraphIdPtrs[InputNode(input)];
for (auto& subgraphId : subgraphIds) {
if (subgraphId == inputId) {
subgraphId = *firstInputSubgraphIdPtr;
}
}
}
subgraphIdPtrs.emplace(node, firstInputSubgraphIdPtr);
}
}
NodeMap<int> result;
for (const auto& subgraphIdPtr : subgraphIdPtrs) {
result.emplace(subgraphIdPtr.first, *(subgraphIdPtr.second));
}
return result;
};
// Split cyclic dependencies.
for (size_t prevSubgraphs = 0, cyclicSplitStep = 0; prevSubgraphs != subgraphInputs.size(); ++cyclicSplitStep) {
OPENVINO_ASSERT(cyclicSplitStep < orderedOps.size(), "Cannot resolve cycles during submodels split");
prevSubgraphs = subgraphInputs.size();
auto subgraphIds = CollectSubgraphs();
// All inputs that belong to the same subgraph as node
std::unordered_map<std::shared_ptr<ov::Node>, InputSet> nodeSubgraphInputDependencies;
// All inputs that depends on the same subgraph as node
std::unordered_map<std::shared_ptr<ov::Node>, InputSet> nodeSubgraphCyclicInputDependencies;
for (const auto& node : orderedOps) {
auto& nodeSubgraphInputDependency = nodeSubgraphInputDependencies[node];
auto allNodeSubgraphInputs = intersection(nodeInputDependencies[node], subgraphInputs);
for (const auto& subgraphInput : allNodeSubgraphInputs) {
if (subgraphIds[node] == subgraphIds[subgraphInput.get_node()->shared_from_this()]) {
nodeSubgraphInputDependency.emplace(subgraphInput);
}
}
auto& nodeSubgraphCyclicInputDependency = nodeSubgraphCyclicInputDependencies[node];
for (const auto& subgraphInput : allNodeSubgraphInputs) {
if (!ov::op::util::is_parameter(subgraphInput.get_node()) &&
!ov::op::util::is_constant(subgraphInput.get_node()) &&
subgraphIds[node] == subgraphIds[InputNode(subgraphInput)]) {
nodeSubgraphCyclicInputDependency.emplace(subgraphInput);
}
}
}
for (const auto& node : orderedOps) {
auto& nodeSubgraphCyclicInputDependency = nodeSubgraphCyclicInputDependencies[node];
if (!nodeSubgraphCyclicInputDependency.empty()) {
// Collect all subgraph inputs that cyclic subgraph output depends on
InputSet cyclicInputsDependencies;
for (const auto& cyclicInput : nodeSubgraphCyclicInputDependency) {
for (const auto& input : nodeSubgraphInputDependencies[InputNode(cyclicInput)]) {
cyclicInputsDependencies.emplace(input);
}
}
for (const auto& input : node->inputs()) {
auto& inputNodeSubgraphCyclicInputDependency =
nodeSubgraphCyclicInputDependencies[InputNode(input)];
auto& inputNodeSubgraphInputDependency = nodeSubgraphInputDependencies[InputNode(input)];
if (!intersects(nodeSubgraphCyclicInputDependency, inputNodeSubgraphCyclicInputDependency) &&
intersects(cyclicInputsDependencies, inputNodeSubgraphInputDependency)) {
subgraphInputs.insert(input);
}
}
}
}
}
auto subgraphIds = CollectSubgraphs();
if (dumpDotFile) {
std::map<std::string, int> map_id;
for (const auto& v : subgraphIds) {
if (dump_dot_file) {
auto subgraph_ids = subgraph_collector.get_subgraph_ids();
std::map<std::string, SubgraphCollector::SubgraphId> map_id;
for (const auto& v : subgraph_ids) {
map_id.emplace(v.first->get_friendly_name(), v.second);
}
ov::hetero::debug::dump_subgraphs(model, queryNetworkResult, map_id);
ov::hetero::debug::dump_subgraphs(model, query_model_result, map_id);
}
// Break graph using insertion of result parameter split
NodeMap<std::shared_ptr<ov::Node>> subgraphParameterToPrevResult;
std::vector<std::shared_ptr<ov::op::v0::Result>> results;
{
std::set<ov::Output<ov::Node>> subgraphOutputs;
for (const auto& input : subgraphInputs) {
if (!ov::op::util::is_parameter(input.get_node()) && !ov::op::util::is_constant(input.get_node())) {
subgraphOutputs.insert(input.get_source_output());
}
}
for (const auto& output : subgraphOutputs) {
auto output_subgraph_id = subgraphIds.at(output.get_node_shared_ptr());
auto inputs = output.get_target_inputs();
// Collect input subsets from other subgraphs. Each subset of inputs belongs to the same subgraph
std::map<int, std::set<ov::Input<ov::Node>>> input_subsets;
for (const auto& input : inputs) {
auto input_subgraph_id = subgraphIds.at(input.get_node()->shared_from_this());
if (output_subgraph_id != input_subgraph_id) {
input_subsets[input_subgraph_id].emplace(input);
}
}
// Avoid duplicate results on the same output port
auto result = std::make_shared<ov::op::v0::Result>(output);
ov::copy_runtime_info(output.get_node_shared_ptr(), result);
subgraphIds.emplace(result, output_subgraph_id);
results.push_back(result);
for (const auto& input_subset : input_subsets) {
// Avoid duplicate parameters in the same subgraph
auto parameter =
std::make_shared<ov::op::v0::Parameter>(output.get_element_type(), output.get_partial_shape());
for (const auto& input : input_subset.second) {
output.remove_target_input(input);
ov::copy_runtime_info(input.get_node()->shared_from_this(), parameter);
input.replace_source_output(parameter->output(0));
subgraphIds.emplace(parameter, input_subset.first);
subgraphParameterToPrevResult.emplace(parameter, result);
}
}
}
}
struct Subgraph {
ov::ResultVector _results;
ov::ParameterVector _parameters;
ov::SinkVector _sinks;
std::string _affinity;
};
std::unordered_map<int, Subgraph> subgraphs;
// Extracts subgraph parameters, results and affinities
for (const auto& subgraphIdPtrValue : subgraphIds) {
auto node = subgraphIdPtrValue.first;
auto& subgraph = subgraphs[subgraphIdPtrValue.second];
if (ov::op::util::is_output(node)) {
subgraph._results.emplace_back(std::dynamic_pointer_cast<ov::op::v0::Result>(node->shared_from_this()));
} else if (ov::op::util::is_parameter(node)) {
subgraph._parameters.emplace_back(
std::dynamic_pointer_cast<ov::op::v0::Parameter>(node->shared_from_this()));
} else if (ov::op::util::is_sink(node)) {
subgraph._sinks.emplace_back(std::dynamic_pointer_cast<ov::op::Sink>(node->shared_from_this()));
}
auto itAffinity = affinities.find(node);
if (itAffinity != affinities.end()) {
subgraph._affinity = itAffinity->second;
}
}
results = {};
// Subgraph topological sort
std::vector<Subgraph> allSubgraphs;
for (const auto& subgraph : subgraphs) {
allSubgraphs.emplace_back(std::move(subgraph.second));
}
std::vector<Subgraph> orderedSubgraphs;
NodeSet prevResults;
size_t subgraphTopoSortsStep = 0;
do {
OPENVINO_ASSERT(subgraphTopoSortsStep < subgraphs.size());
++subgraphTopoSortsStep;
std::vector<Subgraph> newOrderedSubgraphs;
auto IsOrderedSubGraph = [&](const Subgraph& subgraph) {
auto& parameters = subgraph._parameters;
return std::all_of(
parameters.begin(),
parameters.end(),
[&](const ov::ParameterVector::value_type& parameter) {
return (graphInputNodes.find(parameter) != graphInputNodes.end()) ||
(prevResults.find(subgraphParameterToPrevResult[parameter]) != prevResults.end());
});
};
std::remove_copy_if(std::begin(allSubgraphs),
std::end(allSubgraphs),
std::back_inserter(newOrderedSubgraphs),
[&](const Subgraph& subgraph) {
return !IsOrderedSubGraph(subgraph);
});
allSubgraphs.erase(std::remove_if(std::begin(allSubgraphs), std::end(allSubgraphs), IsOrderedSubGraph),
std::end(allSubgraphs));
for (const auto& subgraph : newOrderedSubgraphs) {
for (const auto& result : subgraph._results) {
prevResults.insert(result);
}
}
std::move(std::begin(newOrderedSubgraphs), std::end(newOrderedSubgraphs), std::back_inserter(orderedSubgraphs));
} while (!allSubgraphs.empty());
// Get subgraphs sorted topologically
auto ordered_subgraphs = subgraph_collector.get_ordered_subgraphs();
// Prepare mapping between original inputs/outputs and compiled
// submodels inputs/outputs. Example:
@ -367,37 +134,38 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model
const auto& orig_results = model->get_results();
m_inputs_to_submodels_inputs.resize(orig_parameters.size());
m_outputs_to_submodels_outputs.resize(orig_results.size());
for (size_t id = 0; id < orderedSubgraphs.size(); id++) {
for (size_t i = 0; i < orderedSubgraphs[id]._parameters.size(); i++) {
for (size_t id = 0; id < ordered_subgraphs.size(); id++) {
for (size_t i = 0; i < ordered_subgraphs[id]._parameters.size(); i++) {
for (size_t j = 0; j < orig_parameters.size(); j++)
if (orderedSubgraphs[id]._parameters[i] == orig_parameters[j])
if (ordered_subgraphs[id]._parameters[i] == orig_parameters[j])
m_inputs_to_submodels_inputs[j] = {id, i};
}
for (size_t i = 0; i < orderedSubgraphs[id]._results.size(); i++) {
for (size_t i = 0; i < ordered_subgraphs[id]._results.size(); i++) {
for (size_t j = 0; j < orig_results.size(); j++)
if (orderedSubgraphs[id]._results[i] == orig_results[j])
if (ordered_subgraphs[id]._results[i] == orig_results[j])
m_outputs_to_submodels_outputs[j] = {id, i};
}
}
// Prepare mapping between manually splitted inputs/outputs
// to connect tensors between compiled submodels
for (const auto& kvp : subgraphParameterToPrevResult) {
for (const auto& kvp : subgraph_collector.get_subgraph_parameter_to_prev_result()) {
const auto& intermed_output = kvp.second;
const auto& intermed_input = kvp.first;
for (size_t id = 0; id < orderedSubgraphs.size(); id++) {
const auto& out_it =
std::find(orderedSubgraphs[id]._results.begin(), orderedSubgraphs[id]._results.end(), intermed_output);
if (out_it != orderedSubgraphs[id]._results.end()) {
for (size_t id2 = 0; id2 < orderedSubgraphs.size(); id2++) {
for (size_t id = 0; id < ordered_subgraphs.size(); id++) {
const auto& out_it = std::find(ordered_subgraphs[id]._results.begin(),
ordered_subgraphs[id]._results.end(),
intermed_output);
if (out_it != ordered_subgraphs[id]._results.end()) {
for (size_t id2 = 0; id2 < ordered_subgraphs.size(); id2++) {
if (id2 == id)
continue;
const auto& in_it = std::find(orderedSubgraphs[id2]._parameters.begin(),
orderedSubgraphs[id2]._parameters.end(),
const auto& in_it = std::find(ordered_subgraphs[id2]._parameters.begin(),
ordered_subgraphs[id2]._parameters.end(),
intermed_input);
if (in_it != orderedSubgraphs[id2]._parameters.end()) {
auto out_idx = std::distance(orderedSubgraphs[id]._results.begin(), out_it);
auto in_idx = std::distance(orderedSubgraphs[id2]._parameters.begin(), in_it);
if (in_it != ordered_subgraphs[id2]._parameters.end()) {
auto out_idx = std::distance(ordered_subgraphs[id]._results.begin(), out_it);
auto in_idx = std::distance(ordered_subgraphs[id2]._parameters.begin(), in_it);
m_submodels_input_to_prev_output[{id2, in_idx}] = {id, out_idx};
}
}
@ -405,27 +173,28 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model
}
}
m_compiled_submodels.resize(orderedSubgraphs.size());
std::vector<std::shared_ptr<ov::Model>> subFunctions(orderedSubgraphs.size());
m_compiled_submodels.resize(ordered_subgraphs.size());
std::vector<std::shared_ptr<ov::Model>> submodels(ordered_subgraphs.size());
size_t id = 0;
for (const auto& subgraph : orderedSubgraphs) {
for (const auto& subgraph : ordered_subgraphs) {
m_compiled_submodels[id].device = subgraph._affinity;
subFunctions[id] = std::make_shared<ov::Model>(subgraph._results,
subgraph._sinks,
subgraph._parameters,
m_name + '_' + std::to_string(id));
m_compiled_submodels[id].model = subFunctions[id];
submodels[id] = std::make_shared<ov::Model>(subgraph._results,
subgraph._sinks,
subgraph._parameters,
m_name + '_' + std::to_string(id));
m_compiled_submodels[id].model = submodels[id];
auto metaDevices = get_hetero_plugin()->get_properties_per_device(m_compiled_submodels[id].device,
m_cfg.get_device_properties());
auto meta_devices = get_hetero_plugin()->get_properties_per_device(m_compiled_submodels[id].device,
m_cfg.get_device_properties());
// disable caching for subgraphs, because the whole HETERO model is cached
auto device_config = metaDevices[m_compiled_submodels[id].device];
auto device_config = meta_devices[m_compiled_submodels[id].device];
device_config[ov::cache_dir.name()] = "";
// set exclusive_async_requests in case when model is split
if (orderedSubgraphs.size() > 1) {
if (ordered_subgraphs.size() > 1) {
auto supported_internal_properties =
plugin->get_core()->get_property(m_compiled_submodels[id].device, ov::internal::supported_properties);
get_hetero_plugin()->get_core()->get_property(m_compiled_submodels[id].device,
ov::internal::supported_properties);
if (std::find(supported_internal_properties.begin(),
supported_internal_properties.end(),
ov::internal::exclusive_async_requests) != supported_internal_properties.end()) {
@ -433,9 +202,10 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model
device_config.insert(ov::internal::exclusive_async_requests(true));
}
}
m_compiled_submodels[id].compiled_model = plugin->get_core()->compile_model(m_compiled_submodels[id].model,
m_compiled_submodels[id].device,
device_config);
m_compiled_submodels[id].compiled_model =
get_hetero_plugin()->get_core()->compile_model(m_compiled_submodels[id].model,
m_compiled_submodels[id].device,
device_config);
++id;
}
@ -465,19 +235,20 @@ ov::hetero::CompiledModel::CompiledModel(std::istream& model,
ov::AnyMap properties;
auto heteroConfigsNode = heteroNode.child("hetero_config");
FOREACH_CHILD (heteroConfigNode, heteroConfigsNode, "config") {
// clang-format off
FOREACH_CHILD(heteroConfigNode, heteroConfigsNode, "config") {
properties.emplace(GetStrAttr(heteroConfigNode, "key"), GetStrAttr(heteroConfigNode, "value"));
}
m_cfg = ov::hetero::Configuration(properties, m_cfg);
pugi::xml_node subnetworksNode = heteroNode.child("compiled_submodels");
FOREACH_CHILD (subnetworkNode, subnetworksNode, "compiled_submodel") {
FOREACH_CHILD(subnetworkNode, subnetworksNode, "compiled_submodel") {
auto device = GetStrAttr(subnetworkNode, "device");
auto metaDevices = get_hetero_plugin()->get_properties_per_device(device, m_cfg.get_device_properties());
assert(metaDevices.size() == 1);
auto& loadConfig = metaDevices[device];
auto meta_devices = get_hetero_plugin()->get_properties_per_device(device, m_cfg.get_device_properties());
assert(meta_devices.size() == 1);
auto& loadConfig = meta_devices[device];
ov::SoPtr<ov::ICompiledModel> compiled_model;
std::shared_ptr<ov::Model> ov_model;
@ -512,23 +283,24 @@ ov::hetero::CompiledModel::CompiledModel(std::istream& model,
}
auto inputs_map_node = heteroNode.child("inputs_to_submodels_inputs");
FOREACH_CHILD (xml_node, inputs_map_node, "pair") {
FOREACH_CHILD(xml_node, inputs_map_node, "pair") {
m_inputs_to_submodels_inputs.emplace_back(GetUInt64Attr(xml_node, "submodel_idx"),
GetUInt64Attr(xml_node, "node_idx"));
}
auto outputs_map_node = heteroNode.child("outputs_to_submodels_outputs");
FOREACH_CHILD (xml_node, outputs_map_node, "pair") {
FOREACH_CHILD(xml_node, outputs_map_node, "pair") {
m_outputs_to_submodels_outputs.emplace_back(GetUInt64Attr(xml_node, "submodel_idx"),
GetUInt64Attr(xml_node, "node_idx"));
}
auto submodels_input_to_prev_output_node = heteroNode.child("submodels_input_to_prev_output");
FOREACH_CHILD (xml_node, submodels_input_to_prev_output_node, "record") {
FOREACH_CHILD(xml_node, submodels_input_to_prev_output_node, "record") {
std::pair<uint64_t, uint64_t> in_pair = {GetUInt64Attr(xml_node, "in_submodel_idx"),
GetUInt64Attr(xml_node, "in_node_idx")};
std::pair<uint64_t, uint64_t> out_pair = {GetUInt64Attr(xml_node, "out_submodel_idx"),
GetUInt64Attr(xml_node, "out_node_idx")};
m_submodels_input_to_prev_output.emplace(in_pair, out_pair);
}
// clang-format on
set_inputs_and_outputs();
}

2
src/plugins/hetero/src/compiled_model.hpp
View File

@ -39,6 +39,8 @@ public:
private:
friend class InferRequest;
void compile_model(const std::shared_ptr<ov::Model>& model);
std::shared_ptr<const Plugin> get_hetero_plugin() const;
std::shared_ptr<ov::ISyncInferRequest> create_sync_infer_request() const override;

10
src/plugins/hetero/src/graph_debug_dump.cpp
View File

@ -30,7 +30,7 @@ void dump_affinities(const std::shared_ptr<ov::Model>& model,
const std::map<std::string, std::string>& supported_ops_map,
const std::unordered_set<std::string>& devices) {
auto name = model->get_friendly_name();
// clang-format off
ov::pass::VisualizeTree{
"hetero_affinity_" + name + ".dot",
[&](const ov::Node& node, std::vector<std::string>& attributes) {
@ -38,7 +38,7 @@ void dump_affinities(const std::shared_ptr<ov::Model>& model,
int colorIndex = 0;
for (auto&& device : devices) {
if (device == nodeDevice) {
attributes.push_back(std::string{"fillcolor="} + colors[colorIndex % colors.size()] +
attributes.push_back(std::string {"fillcolor="} + colors[colorIndex % colors.size()] +
" style=filled");
auto itLabel =
std::find_if(std::begin(attributes), std::end(attributes), [](const std::string& str) {
@ -54,17 +54,18 @@ void dump_affinities(const std::shared_ptr<ov::Model>& model,
}
}}
.run_on_model(model);
// clang-format on
}
void dump_subgraphs(const std::shared_ptr<ov::Model>& model,
const std::map<std::string, std::string>& supported_ops_map,
const std::map<std::string, int>& map_id) {
auto name = model->get_friendly_name();
// clang-format off
ov::pass::VisualizeTree{
"hetero_subgraphs_" + name + ".dot",
[&](const ov::Node& node, std::vector<std::string>& attributes) {
attributes.push_back(std::string{"fillcolor="} +
attributes.push_back(std::string {"fillcolor="} +
colors[map_id.at(node.get_friendly_name()) % colors.size()] + " style=filled");
auto itLabel = std::find_if(std::begin(attributes), std::end(attributes), [](const std::string& str) {
return str.find("label") != std::string::npos;
@ -76,6 +77,7 @@ void dump_subgraphs(const std::shared_ptr<ov::Model>& model,
(*itLabel) += label;
}}
.run_on_model(model);
// clang-format on
}
} // namespace debug
} // namespace hetero

3
src/plugins/hetero/src/plugin.cpp
View File

@ -196,6 +196,3 @@ ov::SoPtr<ov::IRemoteContext> ov::hetero::Plugin::create_context(const ov::AnyMa
ov::SoPtr<ov::IRemoteContext> ov::hetero::Plugin::get_default_context(const ov::AnyMap& remote_properties) const {
OPENVINO_NOT_IMPLEMENTED;
}
static const ov::Version version = {CI_BUILD_NUMBER, "openvino_hetero_plugin"};
OV_DEFINE_PLUGIN_CREATE_FUNCTION(ov::hetero::Plugin, version)

283
src/plugins/hetero/src/subgraph_collector.cpp Normal file
View File

@ -0,0 +1,283 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "subgraph_collector.hpp"
#include <deque>
#include "openvino/core/except.hpp"
#include "openvino/core/rt_info.hpp"
#include "openvino/op/util/op_types.hpp"
namespace {
template <typename Set>
static Set intersection(const Set& lhs, const Set& rhs) {
Set result;
const auto& min_size_set = (lhs.size() < rhs.size()) ? lhs : rhs;
const auto& max_size_set = (lhs.size() >= rhs.size()) ? lhs : rhs;
for (auto&& val : min_size_set)
if (max_size_set.find(val) != max_size_set.end())
result.insert(val);
return result;
}
template <typename Set>
static bool intersects(const Set& lhs, const Set& rhs) {
const auto& min_size_set = (lhs.size() < rhs.size()) ? lhs : rhs;
const auto& max_size_set = (lhs.size() >= rhs.size()) ? lhs : rhs;
for (auto&& val : min_size_set)
if (max_size_set.find(val) != max_size_set.end())
return true;
return false;
}
} // namespace
std::shared_ptr<ov::Node> ov::hetero::SubgraphCollector::input_node(
const ov::hetero::SubgraphCollector::Input& input) const {
return input.get_source_output().get_node_shared_ptr();
}
ov::hetero::SubgraphCollector::SubgraphCollector(const std::shared_ptr<ov::Model>& model,
const AffinitiesMap& affinities)
: _ordered_ops{model->get_ordered_ops()},
_affinities{affinities},
_graph_input_nodes{},
_node_input_dependencies{},
_subgraph_inputs{},
_subgraph_parameter_to_prev_result{} {
init();
split_cyclic_dependencies();
_subgraph_ids = collect_subgraphs_ids();
}
void ov::hetero::SubgraphCollector::init() {
// Get all subgraph inputs using just node affinities. Also collect transitive closure
for (const auto& node : _ordered_ops) {
if (ov::op::util::is_parameter(node) || ov::op::util::is_constant(node)) {
_graph_input_nodes.insert(node);
_subgraph_inputs.insert(Input{node.get(), 0});
_node_input_dependencies[node].insert(Input{node.get(), 0});
} else {
auto inputs = node->inputs();
auto& node_input_dependency = _node_input_dependencies[node];
for (const auto& input : inputs) {
node_input_dependency.insert(input);
auto& inputDependency = _node_input_dependencies[input_node(input)];
node_input_dependency.insert(inputDependency.begin(), inputDependency.end());
if (_affinities.at(node) != _affinities.at(input_node(input))) {
_subgraph_inputs.insert(input);
}
}
}
}
}
void ov::hetero::SubgraphCollector::split_cyclic_dependencies() {
// Split cyclic dependencies.
for (size_t prev_subgraphs = 0, cyclic_split_step = 0; prev_subgraphs != _subgraph_inputs.size();
++cyclic_split_step) {
OPENVINO_ASSERT(cyclic_split_step < _ordered_ops.size(), "Cannot resolve cycles during submodels split!");
prev_subgraphs = _subgraph_inputs.size();
auto subgraph_ids = collect_subgraphs_ids();
// All inputs that belong to the same subgraph as node
std::unordered_map<std::shared_ptr<ov::Node>, InputSet> node_subgraph_input_dependencies;
// All inputs that depends on the same subgraph as node
std::unordered_map<std::shared_ptr<ov::Node>, InputSet> node_subgraph_cyclic_iput_dependencies;
for (const auto& node : _ordered_ops) {
auto& node_subgraph_input_dependency = node_subgraph_input_dependencies[node];
auto all_node_subgraph_inputs = intersection(_node_input_dependencies[node], _subgraph_inputs);
for (const auto& subgraphInput : all_node_subgraph_inputs) {
if (subgraph_ids[node] == subgraph_ids[subgraphInput.get_node()->shared_from_this()]) {
node_subgraph_input_dependency.emplace(subgraphInput);
}
}
auto& node_subgraph_cyclic_input_dependency = node_subgraph_cyclic_iput_dependencies[node];
for (const auto& subgraphInput : all_node_subgraph_inputs) {
if (!ov::op::util::is_parameter(subgraphInput.get_node()) &&
!ov::op::util::is_constant(subgraphInput.get_node()) &&
subgraph_ids[node] == subgraph_ids[input_node(subgraphInput)]) {
node_subgraph_cyclic_input_dependency.emplace(subgraphInput);
}
}
}
for (const auto& node : _ordered_ops) {
auto& node_subgraph_cyclic_input_dependency = node_subgraph_cyclic_iput_dependencies[node];
if (!node_subgraph_cyclic_input_dependency.empty()) {
// Collect all subgraph inputs that cyclic subgraph output depends on
InputSet cyclicInputsDependencies;
for (const auto& cyclicInput : node_subgraph_cyclic_input_dependency) {
for (const auto& input : node_subgraph_input_dependencies[input_node(cyclicInput)]) {
cyclicInputsDependencies.emplace(input);
}
}
for (const auto& input : node->inputs()) {
auto& input_node_subgraph_cyclic_input_dependency =
node_subgraph_cyclic_iput_dependencies[input_node(input)];
auto& input_node_subgraph_input_dependency = node_subgraph_input_dependencies[input_node(input)];
if (!intersects(node_subgraph_cyclic_input_dependency,
input_node_subgraph_cyclic_input_dependency) &&
intersects(cyclicInputsDependencies, input_node_subgraph_input_dependency)) {
_subgraph_inputs.insert(input);
}
}
}
}
}
}
ov::hetero::SubgraphCollector::SubgraphIdsMap ov::hetero::SubgraphCollector::collect_subgraphs_ids() {
std::deque<SubgraphId> subgraph_ids;
NodeMap<SubgraphId*> subgraph_id_ptrs;
for (const auto& node : _ordered_ops) {
auto all_node_inputs = node->inputs();
std::vector<Input> inputs;
for (const auto& input : all_node_inputs) {
if (_subgraph_inputs.find(input) == _subgraph_inputs.end()) {
inputs.emplace_back(std::move(input));
}
}
if (inputs.empty()) {
subgraph_ids.push_back(static_cast<SubgraphId>(subgraph_ids.size()));
subgraph_id_ptrs.emplace(node, &(subgraph_ids.back()));
} else {
auto first_input_subgraph_id_ptr = subgraph_id_ptrs[input_node(inputs.front())];
for (const auto& input : inputs) {
auto input_id = *subgraph_id_ptrs[input_node(input)];
for (auto& subgraph_id : subgraph_ids) {
if (subgraph_id == input_id) {
subgraph_id = *first_input_subgraph_id_ptr;
}
}
}
subgraph_id_ptrs.emplace(node, first_input_subgraph_id_ptr);
}
}
SubgraphIdsMap result;
for (const auto& subgraph_id_ptr : subgraph_id_ptrs) {
result.emplace(subgraph_id_ptr.first, *(subgraph_id_ptr.second));
}
return result;
}
void ov::hetero::SubgraphCollector::split_subgraphs_by_parameter_results() {
// Break graph using insertion of result parameter split
std::vector<std::shared_ptr<ov::op::v0::Result>> results;
{
std::set<ov::Output<ov::Node>> subgraph_outputs;
for (const auto& input : _subgraph_inputs) {
if (!ov::op::util::is_parameter(input.get_node()) && !ov::op::util::is_constant(input.get_node())) {
auto input_source_output = input.get_source_output();
if (!ov::op::util::is_parameter(input_source_output.get_node()) &&
!ov::op::util::is_constant(input_source_output.get_node())) {
subgraph_outputs.insert(input_source_output);
}
}
}
for (const auto& output : subgraph_outputs) {
auto output_subgraph_id = _subgraph_ids.at(output.get_node_shared_ptr());
auto inputs = output.get_target_inputs();
// Collect input subsets from other subgraphs. Each subset of inputs belongs to the same subgraph
std::map<int, std::set<ov::Input<ov::Node>>> input_subsets;
for (const auto& input : inputs) {
auto input_subgraph_id = _subgraph_ids.at(input.get_node()->shared_from_this());
if (output_subgraph_id != input_subgraph_id) {
input_subsets[input_subgraph_id].emplace(input);
}
}
// Avoid duplicate results on the same output port
auto result = std::make_shared<ov::op::v0::Result>(output);
const auto name = output.get_node()->get_friendly_name() + "_" + std::to_string(output.get_index());
result->set_friendly_name(name + "_result");
ov::copy_runtime_info(output.get_node_shared_ptr(), result);
_subgraph_ids.emplace(result, output_subgraph_id);
results.push_back(result);
for (const auto& input_subset : input_subsets) {
// Avoid duplicate parameters in the same subgraph
auto parameter =
std::make_shared<ov::op::v0::Parameter>(output.get_element_type(), output.get_partial_shape());
parameter->set_friendly_name(name + "_parameter");
for (const auto& input : input_subset.second) {
output.remove_target_input(input);
ov::copy_runtime_info(input.get_node()->shared_from_this(), parameter);
input.replace_source_output(parameter->output(0));
_subgraph_ids.emplace(parameter, input_subset.first);
_subgraph_parameter_to_prev_result.emplace(parameter, result);
}
}
}
}
}
std::vector<ov::hetero::SubgraphCollector::Subgraph> ov::hetero::SubgraphCollector::get_ordered_subgraphs() {
// Break graph using insertion of result parameter split
split_subgraphs_by_parameter_results();
// Extracts subgraph parameters, results and affinities
auto subgraphs = collect_subgraphs();
// Subgraph topological sort
std::vector<Subgraph> all_subgraphs;
for (const auto& subgraph : subgraphs) {
all_subgraphs.emplace_back(std::move(subgraph.second));
}
std::vector<Subgraph> ordered_subgraphs;
NodeSet prev_results;
size_t subgraph_topo_sorts_step = 0;
do {
OPENVINO_ASSERT(subgraph_topo_sorts_step < subgraphs.size(), "Cannot sort subgraphs!");
++subgraph_topo_sorts_step;
std::vector<Subgraph> new_ordered_subgraphs;
auto is_ordered_subgraph = [&](const Subgraph& subgraph) {
auto& parameters = subgraph._parameters;
return std::all_of(
parameters.begin(),
parameters.end(),
[&](const ov::ParameterVector::value_type& parameter) {
return (_graph_input_nodes.find(parameter) != _graph_input_nodes.end()) ||
(prev_results.find(_subgraph_parameter_to_prev_result[parameter]) != prev_results.end());
});
};
std::remove_copy_if(std::begin(all_subgraphs),
std::end(all_subgraphs),
std::back_inserter(new_ordered_subgraphs),
[&](const Subgraph& subgraph) {
return !is_ordered_subgraph(subgraph);
});
all_subgraphs.erase(std::remove_if(std::begin(all_subgraphs), std::end(all_subgraphs), is_ordered_subgraph),
std::end(all_subgraphs));
for (const auto& subgraph : new_ordered_subgraphs) {
for (const auto& result : subgraph._results) {
prev_results.insert(result);
}
}
std::move(std::begin(new_ordered_subgraphs),
std::end(new_ordered_subgraphs),
std::back_inserter(ordered_subgraphs));
} while (!all_subgraphs.empty());
return ordered_subgraphs;
}
std::unordered_map<ov::hetero::SubgraphCollector::SubgraphId, ov::hetero::SubgraphCollector::Subgraph>
ov::hetero::SubgraphCollector::collect_subgraphs() {
std::unordered_map<SubgraphId, Subgraph> subgraphs;
// Extracts subgraph parameters, results and affinities
for (const auto& subgraph_id_ptr_value : _subgraph_ids) {
auto node = subgraph_id_ptr_value.first;
auto& subgraph = subgraphs[subgraph_id_ptr_value.second];
if (ov::op::util::is_output(node)) {
subgraph._results.emplace_back(ov::as_type_ptr<ov::op::v0::Result>(node->shared_from_this()));
} else if (ov::op::util::is_parameter(node)) {
subgraph._parameters.emplace_back(ov::as_type_ptr<ov::op::v0::Parameter>(node->shared_from_this()));
} else if (ov::op::util::is_sink(node)) {
subgraph._sinks.emplace_back(ov::as_type_ptr<ov::op::Sink>(node->shared_from_this()));
}
auto it_affinity = _affinities.find(node);
if (it_affinity != _affinities.end()) {
subgraph._affinity = it_affinity->second;
}
}
return subgraphs;
}

61
src/plugins/hetero/src/subgraph_collector.hpp Normal file
View File

@ -0,0 +1,61 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <string>
#include <vector>
#include "openvino/runtime/common.hpp"
#include "openvino/runtime/core.hpp"
namespace ov {
namespace hetero {
class SubgraphCollector {
public:
struct Subgraph {
ov::ResultVector _results;
ov::ParameterVector _parameters;
ov::SinkVector _sinks;
std::string _affinity;
};
template <typename T>
using NodeMap = std::unordered_map<std::shared_ptr<ov::Node>, T>;
using NodeSet = std::unordered_set<std::shared_ptr<ov::Node>>;
using AffinitiesMap = NodeMap<std::string>;
using SubgraphId = int;
using SubgraphIdsMap = NodeMap<SubgraphId>;
using ParameterResultMap = NodeMap<std::shared_ptr<ov::Node>>;
using Input = ov::Input<ov::Node>;
using InputSet = std::set<Input>;
SubgraphCollector(const std::shared_ptr<ov::Model>& model, const AffinitiesMap& affinities);
SubgraphIdsMap get_subgraph_ids() {
return _subgraph_ids;
}
ParameterResultMap get_subgraph_parameter_to_prev_result() {
return _subgraph_parameter_to_prev_result;
}
std::vector<Subgraph> get_ordered_subgraphs();
private:
void init();
void split_cyclic_dependencies();
void split_subgraphs_by_parameter_results();
SubgraphIdsMap collect_subgraphs_ids();
std::unordered_map<SubgraphId, Subgraph> collect_subgraphs();
std::shared_ptr<ov::Node> input_node(const Input& input) const;
ov::NodeVector _ordered_ops;
AffinitiesMap _affinities;
NodeSet _graph_input_nodes;
NodeMap<InputSet> _node_input_dependencies;
InputSet _subgraph_inputs;
SubgraphIdsMap _subgraph_ids;
ParameterResultMap _subgraph_parameter_to_prev_result;
};
} // namespace hetero
} // namespace ov

8
src/plugins/hetero/src/version.cpp Normal file
View File

@ -0,0 +1,8 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "plugin.hpp"
static const ov::Version version = {CI_BUILD_NUMBER, "openvino_hetero_plugin"};
OV_DEFINE_PLUGIN_CREATE_FUNCTION(ov::hetero::Plugin, version)

31
src/plugins/hetero/tests/unit/CMakeLists.txt Normal file
View File

@ -0,0 +1,31 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
set(TARGET_NAME ov_hetero_unit_tests)
if(BUILD_SHARED_LIBS)
set (OBJ_LIB $<TARGET_OBJECTS:openvino_hetero_plugin_obj>)
endif()
ov_add_test_target(
NAME
${TARGET_NAME}
ROOT
${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES
PUBLIC
$<TARGET_PROPERTY:openvino_hetero_plugin,SOURCE_DIR>/src
${CMAKE_CURRENT_SOURCE_DIR}
OBJECT_FILES
${OBJ_LIB}
LINK_LIBRARIES
unit_test_utils
ngraphFunctions
DEPENDENCIES
mock_engine
ngraphFunctions
ADD_CLANG_FORMAT
LABELS
HETERO
)

143
src/plugins/hetero/tests/unit/subgraph_collector.cpp Normal file
View File

@ -0,0 +1,143 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "subgraph_collector.hpp"
#include <gtest/gtest.h>
#include "common_test_utils/graph_comparator.hpp"
#include "openvino/core/except.hpp"
#include "openvino/op/ops.hpp"
using namespace ov::hetero;
namespace {
std::shared_ptr<ov::Model> create_test_model() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1});
const_value->set_friendly_name("const_val");
auto add = std::make_shared<ov::op::v1::Add>(param, const_value);
add->set_friendly_name("add");
auto subtract = std::make_shared<ov::op::v1::Subtract>(add, const_value);
subtract->set_friendly_name("sub");
auto reshape_val = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1});
reshape_val->set_friendly_name("reshape_val");
auto reshape = std::make_shared<ov::op::v1::Reshape>(subtract, reshape_val, true);
reshape->set_friendly_name("reshape");
auto result = std::make_shared<ov::op::v0::Result>(reshape);
result->set_friendly_name("res");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
std::shared_ptr<ov::Model> create_subgraph_add() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("input");
auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1});
const_value->set_friendly_name("const_val");
auto add = std::make_shared<ov::op::v1::Add>(param, const_value);
add->set_friendly_name("add");
auto result = std::make_shared<ov::op::v0::Result>(add);
result->set_friendly_name("add_0_result");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
std::shared_ptr<ov::Model> create_subgraph_sub() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("add_0_parameter");
auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1});
const_value->set_friendly_name("const_val");
auto sub = std::make_shared<ov::op::v1::Subtract>(param, const_value);
sub->set_friendly_name("sub");
auto result = std::make_shared<ov::op::v0::Result>(sub);
result->set_friendly_name("sub_0_result");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
std::shared_ptr<ov::Model> create_subgraph_reshape() {
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
param->set_friendly_name("sub_0_parameter");
auto reshape_val = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1});
reshape_val->set_friendly_name("reshape_val");
auto reshape = std::make_shared<ov::op::v1::Reshape>(param, reshape_val, true);
reshape->set_friendly_name("reshape");
auto result = std::make_shared<ov::op::v0::Result>(reshape);
result->set_friendly_name("res");
return std::make_shared<ov::Model>(ov::ResultVector{result}, ov::ParameterVector{param});
}
} // namespace
class SubgraphCollectorTest : public testing::Test {
void SetUp() override {
m_model = create_test_model();
m_submodels = {};
m_submodels.push_back(create_subgraph_add());
m_submodels.push_back(create_subgraph_sub());
m_submodels.push_back(create_subgraph_reshape());
}
void TearDown() override {}
protected:
std::shared_ptr<ov::Model> m_model;
std::vector<std::shared_ptr<ov::Model>> m_submodels;
};
TEST_F(SubgraphCollectorTest, check_subgraphs) {
const std::map<std::string, std::string> supported_ops = {
{"input", "MOCK.0"},
{"const_val", "MOCK.0"},
{"add", "MOCK.0"},
{"sub", "MOCK.1"},
{"reshape_val", "MOCK.0"},
{"reshape", "MOCK.0"},
{"res", "MOCK.0"},
};
const std::map<std::string, SubgraphCollector::SubgraphId> expected_ids = {
{"input", 0},
{"const_val", 0},
{"add", 0},
{"sub", 2},
{"reshape_val", 3},
{"reshape", 3},
{"res", 3},
};
const std::map<std::string, std::string> expected_parameter_to_prev_result_names = {
{"sub_0_parameter", "sub_0_result"},
{"add_0_parameter", "add_0_result"},
};
SubgraphCollector::AffinitiesMap affinities;
SubgraphCollector::SubgraphIdsMap exptected_subgraphs_ids;
const auto ordered_ops = m_model->get_ordered_ops();
for (const auto& node : ordered_ops) {
const auto name = node->get_friendly_name();
OPENVINO_ASSERT(supported_ops.count(name));
affinities[node] = supported_ops.at(name);
OPENVINO_ASSERT(expected_ids.count(name));
exptected_subgraphs_ids[node] = expected_ids.at(name);
}
SubgraphCollector subgraph_collector(m_model, affinities);
auto actual_subgraphs_ids = subgraph_collector.get_subgraph_ids();
for (auto& op : ordered_ops) {
ASSERT_EQ(actual_subgraphs_ids.at(op), exptected_subgraphs_ids.at(op));
}
// Subgraphs are not collected yet
ASSERT_EQ(0, subgraph_collector.get_subgraph_parameter_to_prev_result().size());
// Collect subgraphs
auto actual_subgraphs = subgraph_collector.get_ordered_subgraphs();
const size_t submodels_number = 3;
ASSERT_EQ(submodels_number, actual_subgraphs.size());
auto get_submodel = [&](size_t i) {
return std::make_shared<ov::Model>(actual_subgraphs.at(i)._results, actual_subgraphs.at(i)._parameters);
};
std::pair<bool, std::string> res;
for (size_t i = 0; i < submodels_number; i++) {
auto submodel = get_submodel(i);
auto res = compare_functions(m_submodels.at(i), submodel);
ASSERT_TRUE(res.first) << res.second;
}
auto actual_parameter_to_prev_result = subgraph_collector.get_subgraph_parameter_to_prev_result();
ASSERT_EQ(actual_parameter_to_prev_result.size(), expected_parameter_to_prev_result_names.size());
for (auto& item : actual_parameter_to_prev_result) {
ASSERT_EQ(item.second->get_friendly_name(),
expected_parameter_to_prev_result_names.at(item.first->get_friendly_name()));
}
}