diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index baf12669aa7..91c6ee5814e 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -368,6 +368,9 @@ jobs: - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVProxyTests.xml displayName: 'OV Proxy Plugin Tests' + - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroUnitTests.xml + displayName: 'OV Hetero Unit Tests' + - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml displayName: 'OV Hetero Func Tests' @@ -452,7 +455,7 @@ jobs: --junitxml=$(INSTALL_TEST_DIR)/TEST-Pyngraph.xml \ --ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_utils/test_utils.py displayName: 'Python API 2.0 Tests' - + # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time - script: | python3 -m pytest -sv $(REPO_DIR)/src/frontends/onnx/tests $(PYTHON_STATIC_ARGS) \ diff --git a/.ci/azure/linux_debian.yml b/.ci/azure/linux_debian.yml index c98da83cba7..690ad5c2c95 100644 --- a/.ci/azure/linux_debian.yml +++ b/.ci/azure/linux_debian.yml @@ -284,6 +284,12 @@ jobs: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'OV Proxy Tests' + - script: | + $(INSTALL_TEST_DIR)/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroUnitTests.xml + env: + LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) + displayName: 'OV Hetero Unit Tests' + - script: | $(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml env: @@ -373,7 +379,7 @@ jobs: env: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) PYTHONPATH: $(INSTALL_TEST_DIR) - displayName: 'ONNX Frontend Python Tests' + displayName: 'ONNX Frontend Python Tests' - script: | set -e diff --git a/.ci/azure/mac.yml b/.ci/azure/mac.yml index cd847128d0e..d39b10e536e 100644 --- a/.ci/azure/mac.yml +++ b/.ci/azure/mac.yml @@ -189,6 +189,10 @@ jobs: displayName: 'OV Proxy Plugin Tests' enabled: 'false' + - script: $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroUnitTests.xml + displayName: 'OV Hetero Unit Tests' + enabled: 'false' + - script: $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml displayName: 'OV Hetero Func Tests' enabled: 'false' diff --git a/.ci/azure/windows.yml b/.ci/azure/windows.yml index f51529f2b89..0c1383b4e7c 100644 --- a/.ci/azure/windows.yml +++ b/.ci/azure/windows.yml @@ -264,6 +264,9 @@ jobs: - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVProxyTests.xml displayName: 'OV Proxy Plugin Tests' + - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVHeteroUnitTests.xml + displayName: 'OV Hetero Unit Tests' + - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVHeteroFuncTests.xml displayName: 'OV Hetero Func Tests' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index e5315af9c85..9f9d3d3878f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -460,6 +460,11 @@ jobs: source ${{ env.INSTALL_DIR }}/setupvars.sh ${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml + - name: Hetero Unit Tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml + - name: Hetero Func Tests run: | source ${{ env.INSTALL_DIR }}/setupvars.sh diff --git a/.github/workflows/linux_debian.yml b/.github/workflows/linux_debian.yml index 31f76afaa84..6e65a344764 100644 --- a/.github/workflows/linux_debian.yml +++ b/.github/workflows/linux_debian.yml @@ -86,39 +86,39 @@ jobs: run: | sudo -E apt update sudo -E ${{ env.OPENVINO_REPO }}/install_build_dependencies.sh - + # 'clang' is used as a default compiler sudo apt --assume-yes install clang sudo apt --assume-yes install --no-install-recommends libopencv-imgproc-dev libopencv-imgcodecs-dev - + # Speed up build sudo apt -y --no-install-recommends install unzip wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip unzip ninja-linux.zip sudo cp -v ninja /usr/local/bin/ - + # Speed up tests git clone https://github.com/google/gtest-parallel.git - name: Install python dependencies run: | python3 -m pip install --upgrade pip - + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/wheel/requirements-dev.txt python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements.txt - + # For running Python API tests python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt - + # For running Paddle frontend unit tests python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt - + # For running ONNX frontend unit tests python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests/requirements.txt - + # For running TensorFlow frontend unit tests python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/tensorflow/tests/requirements.txt - + # For MO unit tests python3 -m pip install -U pip python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_mxnet.txt @@ -128,7 +128,7 @@ jobs: python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_tf2.txt python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt - + # for Python API tests python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements_test.txt python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements.txt @@ -260,6 +260,11 @@ jobs: export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH ${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml + - name: Hetero Unit Tests + run: | + export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH + ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml + - name: Hetero Func Tests run: | export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH @@ -344,7 +349,7 @@ jobs: run: | # For python imports to import pybind_mock_frontend export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:${{ env.OPENVINO_REPO }}/tools/mo:$PYTHONPATH - + export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino \ @@ -355,7 +360,7 @@ jobs: run: | # For python imports to import pybind_mock_frontend export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:${{ env.OPENVINO_REPO }}/tools/mo:$PYTHONPATH - + export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH python3 -m pytest -sv ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests \ @@ -392,7 +397,7 @@ jobs: - name: Samples Smoke Tests run: | python3 -m pip install --ignore-installed PyYAML -r ${{ env.INSTALL_TEST_DIR }}/smoke_tests/requirements.txt - + export LD_LIBRARY_PATH=${{ env.IE_APP_PATH }}:$LD_LIBRARY_PATH python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/smoke_tests -k "not GNA" \ @@ -409,14 +414,14 @@ jobs: run: | python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH - + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_Roll.xml - name: TensorFlow Lite Layer Tests - TFL FE run: | python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH - + # Need to be reinstalled to have correct numpy version python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_caffe.txt python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_kaldi.txt @@ -424,7 +429,7 @@ jobs: python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_tf2.txt python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_mxnet.txt - + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_lite_tests/ --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tfl_fe.xml env: TEST_DEVICE: CPU diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 66732d000f9..a6c775c42c2 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -645,6 +645,11 @@ jobs: run: | call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml + - name: Hetero Unit Tests + shell: cmd + run: | + call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml + - name: Hetero Func Tests shell: cmd run: | diff --git a/src/plugins/hetero/CMakeLists.txt b/src/plugins/hetero/CMakeLists.txt index e2866661e8e..ca8b8b21a97 100644 --- a/src/plugins/hetero/CMakeLists.txt +++ b/src/plugins/hetero/CMakeLists.txt @@ -6,7 +6,7 @@ if (NOT ENABLE_HETERO) return() endif() -set (TARGET_NAME "openvino_hetero_plugin") +set(TARGET_NAME openvino_hetero_plugin) file(GLOB SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp) file(GLOB HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp) @@ -15,7 +15,7 @@ ov_add_plugin(NAME ${TARGET_NAME} DEVICE_NAME "HETERO" PSEUDO_DEVICE SOURCES ${SOURCES} ${HEADERS} - VERSION_DEFINES_FOR src/plugin.cpp + VERSION_DEFINES_FOR src/version.cpp ADD_CLANG_FORMAT) ie_faster_build(${TARGET_NAME} @@ -29,6 +29,38 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) +if(BUILD_SHARED_LIBS) + set(OBJ_NAME ${TARGET_NAME}_obj) + + add_library(${OBJ_NAME} OBJECT ${SOURCES} ${HEADERS}) + link_system_libraries(${OBJ_NAME} PUBLIC openvino::pugixml) + + ov_add_version_defines(src/version.cpp ${OBJ_NAME}) + + target_include_directories(${OBJ_NAME} + PRIVATE + $ + $ + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/src + $) + + set_ie_threading_interface_for(${OBJ_NAME}) + + target_compile_definitions(${OBJ_NAME} + PRIVATE + USE_STATIC_IE IMPLEMENT_INFERENCE_ENGINE_PLUGIN IMPLEMENT_INFERENCE_EXTENSION_API + $ + $) + + set_target_properties(${TARGET_NAME}_obj PROPERTIES EXCLUDE_FROM_ALL ON) + set_target_properties(${TARGET_NAME}_obj PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) +endif() + +if(ENABLE_TESTS) + add_subdirectory(tests/unit) +endif() + if(ENABLE_FUNCTIONAL_TESTS) add_subdirectory(tests/functional) endif() \ No newline at end of file diff --git a/src/plugins/hetero/src/compiled_model.cpp b/src/plugins/hetero/src/compiled_model.cpp index 7b8bf399286..73824c88189 100644 --- a/src/plugins/hetero/src/compiled_model.cpp +++ b/src/plugins/hetero/src/compiled_model.cpp @@ -18,35 +18,9 @@ #include "openvino/util/common_util.hpp" #include "plugin.hpp" #include "properties.hpp" +#include "subgraph_collector.hpp" #include "xml_parse_utils.h" -template -using NodeMap = std::unordered_map, T>; - -namespace { - -template -static Set intersection(const Set& lhs, const Set& rhs) { - Set result; - const auto& minSizeSet = (lhs.size() < rhs.size()) ? lhs : rhs; - const auto& maxSizeSet = (lhs.size() >= rhs.size()) ? lhs : rhs; - for (auto&& val : minSizeSet) - if (maxSizeSet.find(val) != maxSizeSet.end()) - result.insert(val); - return result; -} - -template -static bool intersects(const Set& lhs, const Set& rhs) { - const auto& minSizeSet = (lhs.size() < rhs.size()) ? lhs : rhs; - const auto& maxSizeSet = (lhs.size() >= rhs.size()) ? lhs : rhs; - for (auto&& val : minSizeSet) - if (maxSizeSet.find(val) != maxSizeSet.end()) - return true; - return false; -} -} // namespace - ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr& model, const std::shared_ptr& plugin, const Configuration& cfg) @@ -54,9 +28,19 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr& model m_cfg(cfg), m_name(model->get_friendly_name()), m_loaded_from_cache(false) { - bool dumpDotFile = m_cfg.dump_graph; + try { + compile_model(model); + } catch (const std::exception& e) { + OPENVINO_THROW("Standard exception from compilation library: ", e.what()); + } catch (...) { + OPENVINO_THROW("Generic exception is thrown"); + } +} + +void ov::hetero::CompiledModel::compile_model(const std::shared_ptr& model) { + bool dump_dot_file = m_cfg.dump_graph; if (std::getenv("OPENVINO_HETERO_VISUALIZE")) - dumpDotFile = true; + dump_dot_file = true; // Calling of ConstantFolding in HETERO plugin is required because // in some cases topology split is happening after constant subgraph. @@ -66,47 +50,39 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr& model manager.register_pass(); manager.run_passes(model); - ov::SupportedOpsMap queryNetworkResult; - auto orderedOps = model->get_ordered_ops(); + ov::SupportedOpsMap query_model_result; + auto ordered_ops = model->get_ordered_ops(); - bool allEmpty = true; + bool all_empty = true; // Get user defined affinity - for (const auto& node : orderedOps) { - auto& nodeInfo = node->get_rt_info(); - auto itInfo = nodeInfo.find("affinity"); - if (itInfo != nodeInfo.end()) { - OPENVINO_ASSERT(itInfo->second.is(), "Unexpected type of \"affinity\" attribute"); - queryNetworkResult.emplace(node->get_friendly_name(), itInfo->second.as()); - allEmpty = false; + for (const auto& node : ordered_ops) { + auto& node_info = node->get_rt_info(); + auto it_info = node_info.find("affinity"); + if (it_info != node_info.end()) { + OPENVINO_ASSERT(it_info->second.is(), "Unexpected type of \"affinity\" attribute"); + query_model_result.emplace(node->get_friendly_name(), it_info->second.as()); + all_empty = false; } } - if (queryNetworkResult.empty()) { + if (query_model_result.empty()) { // Restore properties in order to pass "device priorities" together // with devices properties auto full_properties = m_cfg.get_hetero_properties(); for (const auto& property : m_cfg.get_device_properties()) full_properties[property.first] = property.second; - queryNetworkResult = plugin->query_model(model, full_properties); + query_model_result = get_hetero_plugin()->query_model(model, full_properties); } - using Input = ov::Input; - using NodeSet = std::unordered_set>; - using InputSet = std::set; - - auto InputNode = [](const Input& input) { - return input.get_source_output().get_node_shared_ptr(); - }; - std::unordered_set devices; - NodeMap affinities; + SubgraphCollector::AffinitiesMap affinities; // Check that all nodes has user or plugin defined affinities - for (const auto& node : orderedOps) { - auto itAffinity = queryNetworkResult.find(node->get_friendly_name()); - if (itAffinity != queryNetworkResult.end()) { - affinities[node] = itAffinity->second; - devices.emplace(itAffinity->second); - } else if (allEmpty) { + for (const auto& node : ordered_ops) { + auto it_affinity = query_model_result.find(node->get_friendly_name()); + if (it_affinity != query_model_result.end()) { + affinities[node] = it_affinity->second; + devices.emplace(it_affinity->second); + } else if (all_empty) { OPENVINO_THROW("Hetero device used default fallback policy, but some layers eg: \n(Name:", node->get_friendly_name(), ", Type: ", @@ -126,233 +102,24 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr& model } } - if (dumpDotFile) { - ov::hetero::debug::dump_affinities(model, queryNetworkResult, devices); + if (dump_dot_file) { + ov::hetero::debug::dump_affinities(model, query_model_result, devices); } - NodeMap nodeInputDependencies; - NodeSet graphInputNodes; - InputSet subgraphInputs; - // Get all subgraph inputs using just node affinities. Also collect transitive closure - for (const auto& node : orderedOps) { - if (ov::op::util::is_parameter(node) || ov::op::util::is_constant(node)) { - graphInputNodes.insert(node); - subgraphInputs.insert(Input{node.get(), 0}); - nodeInputDependencies[node].insert(Input{node.get(), 0}); - } else { - auto inputs = node->inputs(); - auto& nodeInputDependency = nodeInputDependencies[node]; - for (const auto& input : inputs) { - nodeInputDependency.insert(input); - auto& inputDependency = nodeInputDependencies[InputNode(input)]; - nodeInputDependency.insert(inputDependency.begin(), inputDependency.end()); - if (affinities[node] != affinities[InputNode(input)]) { - subgraphInputs.insert(input); - } - } - } - } + // Init subgraph collector + SubgraphCollector subgraph_collector(model, affinities); - // Assign each node subgraph ID - auto CollectSubgraphs = [&] { - std::deque subgraphIds; - NodeMap subgraphIdPtrs; - for (const auto& node : orderedOps) { - auto allNodeInputs = node->inputs(); - std::vector inputs; - for (const auto& input : allNodeInputs) { - if (subgraphInputs.find(input) == subgraphInputs.end()) { - inputs.emplace_back(std::move(input)); - } - } - if (inputs.empty()) { - subgraphIds.push_back(static_cast(subgraphIds.size())); - subgraphIdPtrs.emplace(node, &(subgraphIds.back())); - } else { - auto firstInputSubgraphIdPtr = subgraphIdPtrs[InputNode(inputs.front())]; - for (const auto& input : inputs) { - auto inputId = *subgraphIdPtrs[InputNode(input)]; - for (auto& subgraphId : subgraphIds) { - if (subgraphId == inputId) { - subgraphId = *firstInputSubgraphIdPtr; - } - } - } - subgraphIdPtrs.emplace(node, firstInputSubgraphIdPtr); - } - } - NodeMap result; - for (const auto& subgraphIdPtr : subgraphIdPtrs) { - result.emplace(subgraphIdPtr.first, *(subgraphIdPtr.second)); - } - return result; - }; - - // Split cyclic dependencies. - for (size_t prevSubgraphs = 0, cyclicSplitStep = 0; prevSubgraphs != subgraphInputs.size(); ++cyclicSplitStep) { - OPENVINO_ASSERT(cyclicSplitStep < orderedOps.size(), "Cannot resolve cycles during submodels split"); - prevSubgraphs = subgraphInputs.size(); - auto subgraphIds = CollectSubgraphs(); - // All inputs that belong to the same subgraph as node - std::unordered_map, InputSet> nodeSubgraphInputDependencies; - // All inputs that depends on the same subgraph as node - std::unordered_map, InputSet> nodeSubgraphCyclicInputDependencies; - for (const auto& node : orderedOps) { - auto& nodeSubgraphInputDependency = nodeSubgraphInputDependencies[node]; - auto allNodeSubgraphInputs = intersection(nodeInputDependencies[node], subgraphInputs); - for (const auto& subgraphInput : allNodeSubgraphInputs) { - if (subgraphIds[node] == subgraphIds[subgraphInput.get_node()->shared_from_this()]) { - nodeSubgraphInputDependency.emplace(subgraphInput); - } - } - auto& nodeSubgraphCyclicInputDependency = nodeSubgraphCyclicInputDependencies[node]; - for (const auto& subgraphInput : allNodeSubgraphInputs) { - if (!ov::op::util::is_parameter(subgraphInput.get_node()) && - !ov::op::util::is_constant(subgraphInput.get_node()) && - subgraphIds[node] == subgraphIds[InputNode(subgraphInput)]) { - nodeSubgraphCyclicInputDependency.emplace(subgraphInput); - } - } - } - - for (const auto& node : orderedOps) { - auto& nodeSubgraphCyclicInputDependency = nodeSubgraphCyclicInputDependencies[node]; - if (!nodeSubgraphCyclicInputDependency.empty()) { - // Collect all subgraph inputs that cyclic subgraph output depends on - InputSet cyclicInputsDependencies; - for (const auto& cyclicInput : nodeSubgraphCyclicInputDependency) { - for (const auto& input : nodeSubgraphInputDependencies[InputNode(cyclicInput)]) { - cyclicInputsDependencies.emplace(input); - } - } - for (const auto& input : node->inputs()) { - auto& inputNodeSubgraphCyclicInputDependency = - nodeSubgraphCyclicInputDependencies[InputNode(input)]; - auto& inputNodeSubgraphInputDependency = nodeSubgraphInputDependencies[InputNode(input)]; - if (!intersects(nodeSubgraphCyclicInputDependency, inputNodeSubgraphCyclicInputDependency) && - intersects(cyclicInputsDependencies, inputNodeSubgraphInputDependency)) { - subgraphInputs.insert(input); - } - } - } - } - } - - auto subgraphIds = CollectSubgraphs(); - - if (dumpDotFile) { - std::map map_id; - for (const auto& v : subgraphIds) { + if (dump_dot_file) { + auto subgraph_ids = subgraph_collector.get_subgraph_ids(); + std::map map_id; + for (const auto& v : subgraph_ids) { map_id.emplace(v.first->get_friendly_name(), v.second); } - ov::hetero::debug::dump_subgraphs(model, queryNetworkResult, map_id); + ov::hetero::debug::dump_subgraphs(model, query_model_result, map_id); } - // Break graph using insertion of result parameter split - NodeMap> subgraphParameterToPrevResult; - std::vector> results; - { - std::set> subgraphOutputs; - for (const auto& input : subgraphInputs) { - if (!ov::op::util::is_parameter(input.get_node()) && !ov::op::util::is_constant(input.get_node())) { - subgraphOutputs.insert(input.get_source_output()); - } - } - for (const auto& output : subgraphOutputs) { - auto output_subgraph_id = subgraphIds.at(output.get_node_shared_ptr()); - auto inputs = output.get_target_inputs(); - // Collect input subsets from other subgraphs. Each subset of inputs belongs to the same subgraph - std::map>> input_subsets; - for (const auto& input : inputs) { - auto input_subgraph_id = subgraphIds.at(input.get_node()->shared_from_this()); - if (output_subgraph_id != input_subgraph_id) { - input_subsets[input_subgraph_id].emplace(input); - } - } - // Avoid duplicate results on the same output port - auto result = std::make_shared(output); - ov::copy_runtime_info(output.get_node_shared_ptr(), result); - subgraphIds.emplace(result, output_subgraph_id); - results.push_back(result); - for (const auto& input_subset : input_subsets) { - // Avoid duplicate parameters in the same subgraph - auto parameter = - std::make_shared(output.get_element_type(), output.get_partial_shape()); - for (const auto& input : input_subset.second) { - output.remove_target_input(input); - ov::copy_runtime_info(input.get_node()->shared_from_this(), parameter); - input.replace_source_output(parameter->output(0)); - subgraphIds.emplace(parameter, input_subset.first); - subgraphParameterToPrevResult.emplace(parameter, result); - } - } - } - } - - struct Subgraph { - ov::ResultVector _results; - ov::ParameterVector _parameters; - ov::SinkVector _sinks; - std::string _affinity; - }; - std::unordered_map subgraphs; - // Extracts subgraph parameters, results and affinities - for (const auto& subgraphIdPtrValue : subgraphIds) { - auto node = subgraphIdPtrValue.first; - auto& subgraph = subgraphs[subgraphIdPtrValue.second]; - if (ov::op::util::is_output(node)) { - subgraph._results.emplace_back(std::dynamic_pointer_cast(node->shared_from_this())); - } else if (ov::op::util::is_parameter(node)) { - subgraph._parameters.emplace_back( - std::dynamic_pointer_cast(node->shared_from_this())); - } else if (ov::op::util::is_sink(node)) { - subgraph._sinks.emplace_back(std::dynamic_pointer_cast(node->shared_from_this())); - } - auto itAffinity = affinities.find(node); - if (itAffinity != affinities.end()) { - subgraph._affinity = itAffinity->second; - } - } - results = {}; - - // Subgraph topological sort - std::vector allSubgraphs; - for (const auto& subgraph : subgraphs) { - allSubgraphs.emplace_back(std::move(subgraph.second)); - } - - std::vector orderedSubgraphs; - NodeSet prevResults; - size_t subgraphTopoSortsStep = 0; - do { - OPENVINO_ASSERT(subgraphTopoSortsStep < subgraphs.size()); - ++subgraphTopoSortsStep; - std::vector newOrderedSubgraphs; - auto IsOrderedSubGraph = [&](const Subgraph& subgraph) { - auto& parameters = subgraph._parameters; - return std::all_of( - parameters.begin(), - parameters.end(), - [&](const ov::ParameterVector::value_type& parameter) { - return (graphInputNodes.find(parameter) != graphInputNodes.end()) || - (prevResults.find(subgraphParameterToPrevResult[parameter]) != prevResults.end()); - }); - }; - std::remove_copy_if(std::begin(allSubgraphs), - std::end(allSubgraphs), - std::back_inserter(newOrderedSubgraphs), - [&](const Subgraph& subgraph) { - return !IsOrderedSubGraph(subgraph); - }); - allSubgraphs.erase(std::remove_if(std::begin(allSubgraphs), std::end(allSubgraphs), IsOrderedSubGraph), - std::end(allSubgraphs)); - for (const auto& subgraph : newOrderedSubgraphs) { - for (const auto& result : subgraph._results) { - prevResults.insert(result); - } - } - std::move(std::begin(newOrderedSubgraphs), std::end(newOrderedSubgraphs), std::back_inserter(orderedSubgraphs)); - } while (!allSubgraphs.empty()); + // Get subgraphs sorted topologically + auto ordered_subgraphs = subgraph_collector.get_ordered_subgraphs(); // Prepare mapping between original inputs/outputs and compiled // submodels inputs/outputs. Example: @@ -367,37 +134,38 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr& model const auto& orig_results = model->get_results(); m_inputs_to_submodels_inputs.resize(orig_parameters.size()); m_outputs_to_submodels_outputs.resize(orig_results.size()); - for (size_t id = 0; id < orderedSubgraphs.size(); id++) { - for (size_t i = 0; i < orderedSubgraphs[id]._parameters.size(); i++) { + for (size_t id = 0; id < ordered_subgraphs.size(); id++) { + for (size_t i = 0; i < ordered_subgraphs[id]._parameters.size(); i++) { for (size_t j = 0; j < orig_parameters.size(); j++) - if (orderedSubgraphs[id]._parameters[i] == orig_parameters[j]) + if (ordered_subgraphs[id]._parameters[i] == orig_parameters[j]) m_inputs_to_submodels_inputs[j] = {id, i}; } - for (size_t i = 0; i < orderedSubgraphs[id]._results.size(); i++) { + for (size_t i = 0; i < ordered_subgraphs[id]._results.size(); i++) { for (size_t j = 0; j < orig_results.size(); j++) - if (orderedSubgraphs[id]._results[i] == orig_results[j]) + if (ordered_subgraphs[id]._results[i] == orig_results[j]) m_outputs_to_submodels_outputs[j] = {id, i}; } } // Prepare mapping between manually splitted inputs/outputs // to connect tensors between compiled submodels - for (const auto& kvp : subgraphParameterToPrevResult) { + for (const auto& kvp : subgraph_collector.get_subgraph_parameter_to_prev_result()) { const auto& intermed_output = kvp.second; const auto& intermed_input = kvp.first; - for (size_t id = 0; id < orderedSubgraphs.size(); id++) { - const auto& out_it = - std::find(orderedSubgraphs[id]._results.begin(), orderedSubgraphs[id]._results.end(), intermed_output); - if (out_it != orderedSubgraphs[id]._results.end()) { - for (size_t id2 = 0; id2 < orderedSubgraphs.size(); id2++) { + for (size_t id = 0; id < ordered_subgraphs.size(); id++) { + const auto& out_it = std::find(ordered_subgraphs[id]._results.begin(), + ordered_subgraphs[id]._results.end(), + intermed_output); + if (out_it != ordered_subgraphs[id]._results.end()) { + for (size_t id2 = 0; id2 < ordered_subgraphs.size(); id2++) { if (id2 == id) continue; - const auto& in_it = std::find(orderedSubgraphs[id2]._parameters.begin(), - orderedSubgraphs[id2]._parameters.end(), + const auto& in_it = std::find(ordered_subgraphs[id2]._parameters.begin(), + ordered_subgraphs[id2]._parameters.end(), intermed_input); - if (in_it != orderedSubgraphs[id2]._parameters.end()) { - auto out_idx = std::distance(orderedSubgraphs[id]._results.begin(), out_it); - auto in_idx = std::distance(orderedSubgraphs[id2]._parameters.begin(), in_it); + if (in_it != ordered_subgraphs[id2]._parameters.end()) { + auto out_idx = std::distance(ordered_subgraphs[id]._results.begin(), out_it); + auto in_idx = std::distance(ordered_subgraphs[id2]._parameters.begin(), in_it); m_submodels_input_to_prev_output[{id2, in_idx}] = {id, out_idx}; } } @@ -405,27 +173,28 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr& model } } - m_compiled_submodels.resize(orderedSubgraphs.size()); - std::vector> subFunctions(orderedSubgraphs.size()); + m_compiled_submodels.resize(ordered_subgraphs.size()); + std::vector> submodels(ordered_subgraphs.size()); size_t id = 0; - for (const auto& subgraph : orderedSubgraphs) { + for (const auto& subgraph : ordered_subgraphs) { m_compiled_submodels[id].device = subgraph._affinity; - subFunctions[id] = std::make_shared(subgraph._results, - subgraph._sinks, - subgraph._parameters, - m_name + '_' + std::to_string(id)); - m_compiled_submodels[id].model = subFunctions[id]; + submodels[id] = std::make_shared(subgraph._results, + subgraph._sinks, + subgraph._parameters, + m_name + '_' + std::to_string(id)); + m_compiled_submodels[id].model = submodels[id]; - auto metaDevices = get_hetero_plugin()->get_properties_per_device(m_compiled_submodels[id].device, - m_cfg.get_device_properties()); + auto meta_devices = get_hetero_plugin()->get_properties_per_device(m_compiled_submodels[id].device, + m_cfg.get_device_properties()); // disable caching for subgraphs, because the whole HETERO model is cached - auto device_config = metaDevices[m_compiled_submodels[id].device]; + auto device_config = meta_devices[m_compiled_submodels[id].device]; device_config[ov::cache_dir.name()] = ""; // set exclusive_async_requests in case when model is split - if (orderedSubgraphs.size() > 1) { + if (ordered_subgraphs.size() > 1) { auto supported_internal_properties = - plugin->get_core()->get_property(m_compiled_submodels[id].device, ov::internal::supported_properties); + get_hetero_plugin()->get_core()->get_property(m_compiled_submodels[id].device, + ov::internal::supported_properties); if (std::find(supported_internal_properties.begin(), supported_internal_properties.end(), ov::internal::exclusive_async_requests) != supported_internal_properties.end()) { @@ -433,9 +202,10 @@ ov::hetero::CompiledModel::CompiledModel(const std::shared_ptr& model device_config.insert(ov::internal::exclusive_async_requests(true)); } } - m_compiled_submodels[id].compiled_model = plugin->get_core()->compile_model(m_compiled_submodels[id].model, - m_compiled_submodels[id].device, - device_config); + m_compiled_submodels[id].compiled_model = + get_hetero_plugin()->get_core()->compile_model(m_compiled_submodels[id].model, + m_compiled_submodels[id].device, + device_config); ++id; } @@ -465,19 +235,20 @@ ov::hetero::CompiledModel::CompiledModel(std::istream& model, ov::AnyMap properties; auto heteroConfigsNode = heteroNode.child("hetero_config"); - FOREACH_CHILD (heteroConfigNode, heteroConfigsNode, "config") { + // clang-format off + FOREACH_CHILD(heteroConfigNode, heteroConfigsNode, "config") { properties.emplace(GetStrAttr(heteroConfigNode, "key"), GetStrAttr(heteroConfigNode, "value")); } m_cfg = ov::hetero::Configuration(properties, m_cfg); pugi::xml_node subnetworksNode = heteroNode.child("compiled_submodels"); - FOREACH_CHILD (subnetworkNode, subnetworksNode, "compiled_submodel") { + FOREACH_CHILD(subnetworkNode, subnetworksNode, "compiled_submodel") { auto device = GetStrAttr(subnetworkNode, "device"); - auto metaDevices = get_hetero_plugin()->get_properties_per_device(device, m_cfg.get_device_properties()); - assert(metaDevices.size() == 1); - auto& loadConfig = metaDevices[device]; + auto meta_devices = get_hetero_plugin()->get_properties_per_device(device, m_cfg.get_device_properties()); + assert(meta_devices.size() == 1); + auto& loadConfig = meta_devices[device]; ov::SoPtr compiled_model; std::shared_ptr ov_model; @@ -512,23 +283,24 @@ ov::hetero::CompiledModel::CompiledModel(std::istream& model, } auto inputs_map_node = heteroNode.child("inputs_to_submodels_inputs"); - FOREACH_CHILD (xml_node, inputs_map_node, "pair") { + FOREACH_CHILD(xml_node, inputs_map_node, "pair") { m_inputs_to_submodels_inputs.emplace_back(GetUInt64Attr(xml_node, "submodel_idx"), GetUInt64Attr(xml_node, "node_idx")); } auto outputs_map_node = heteroNode.child("outputs_to_submodels_outputs"); - FOREACH_CHILD (xml_node, outputs_map_node, "pair") { + FOREACH_CHILD(xml_node, outputs_map_node, "pair") { m_outputs_to_submodels_outputs.emplace_back(GetUInt64Attr(xml_node, "submodel_idx"), GetUInt64Attr(xml_node, "node_idx")); } auto submodels_input_to_prev_output_node = heteroNode.child("submodels_input_to_prev_output"); - FOREACH_CHILD (xml_node, submodels_input_to_prev_output_node, "record") { + FOREACH_CHILD(xml_node, submodels_input_to_prev_output_node, "record") { std::pair in_pair = {GetUInt64Attr(xml_node, "in_submodel_idx"), GetUInt64Attr(xml_node, "in_node_idx")}; std::pair out_pair = {GetUInt64Attr(xml_node, "out_submodel_idx"), GetUInt64Attr(xml_node, "out_node_idx")}; m_submodels_input_to_prev_output.emplace(in_pair, out_pair); } + // clang-format on set_inputs_and_outputs(); } diff --git a/src/plugins/hetero/src/compiled_model.hpp b/src/plugins/hetero/src/compiled_model.hpp index d264e0ecf96..3acca0a8be4 100644 --- a/src/plugins/hetero/src/compiled_model.hpp +++ b/src/plugins/hetero/src/compiled_model.hpp @@ -39,6 +39,8 @@ public: private: friend class InferRequest; + void compile_model(const std::shared_ptr& model); + std::shared_ptr get_hetero_plugin() const; std::shared_ptr create_sync_infer_request() const override; diff --git a/src/plugins/hetero/src/graph_debug_dump.cpp b/src/plugins/hetero/src/graph_debug_dump.cpp index c773a2d156b..c4d0b755d60 100644 --- a/src/plugins/hetero/src/graph_debug_dump.cpp +++ b/src/plugins/hetero/src/graph_debug_dump.cpp @@ -30,7 +30,7 @@ void dump_affinities(const std::shared_ptr& model, const std::map& supported_ops_map, const std::unordered_set& devices) { auto name = model->get_friendly_name(); - + // clang-format off ov::pass::VisualizeTree{ "hetero_affinity_" + name + ".dot", [&](const ov::Node& node, std::vector& attributes) { @@ -38,7 +38,7 @@ void dump_affinities(const std::shared_ptr& model, int colorIndex = 0; for (auto&& device : devices) { if (device == nodeDevice) { - attributes.push_back(std::string{"fillcolor="} + colors[colorIndex % colors.size()] + + attributes.push_back(std::string {"fillcolor="} + colors[colorIndex % colors.size()] + " style=filled"); auto itLabel = std::find_if(std::begin(attributes), std::end(attributes), [](const std::string& str) { @@ -54,17 +54,18 @@ void dump_affinities(const std::shared_ptr& model, } }} .run_on_model(model); + // clang-format on } void dump_subgraphs(const std::shared_ptr& model, const std::map& supported_ops_map, const std::map& map_id) { auto name = model->get_friendly_name(); - + // clang-format off ov::pass::VisualizeTree{ "hetero_subgraphs_" + name + ".dot", [&](const ov::Node& node, std::vector& attributes) { - attributes.push_back(std::string{"fillcolor="} + + attributes.push_back(std::string {"fillcolor="} + colors[map_id.at(node.get_friendly_name()) % colors.size()] + " style=filled"); auto itLabel = std::find_if(std::begin(attributes), std::end(attributes), [](const std::string& str) { return str.find("label") != std::string::npos; @@ -76,6 +77,7 @@ void dump_subgraphs(const std::shared_ptr& model, (*itLabel) += label; }} .run_on_model(model); + // clang-format on } } // namespace debug } // namespace hetero diff --git a/src/plugins/hetero/src/plugin.cpp b/src/plugins/hetero/src/plugin.cpp index 808c4bc2802..0fb907e7b00 100644 --- a/src/plugins/hetero/src/plugin.cpp +++ b/src/plugins/hetero/src/plugin.cpp @@ -196,6 +196,3 @@ ov::SoPtr ov::hetero::Plugin::create_context(const ov::AnyMa ov::SoPtr ov::hetero::Plugin::get_default_context(const ov::AnyMap& remote_properties) const { OPENVINO_NOT_IMPLEMENTED; } - -static const ov::Version version = {CI_BUILD_NUMBER, "openvino_hetero_plugin"}; -OV_DEFINE_PLUGIN_CREATE_FUNCTION(ov::hetero::Plugin, version) diff --git a/src/plugins/hetero/src/subgraph_collector.cpp b/src/plugins/hetero/src/subgraph_collector.cpp new file mode 100644 index 00000000000..aabb1f068f3 --- /dev/null +++ b/src/plugins/hetero/src/subgraph_collector.cpp @@ -0,0 +1,283 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "subgraph_collector.hpp" + +#include + +#include "openvino/core/except.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/op/util/op_types.hpp" + +namespace { + +template +static Set intersection(const Set& lhs, const Set& rhs) { + Set result; + const auto& min_size_set = (lhs.size() < rhs.size()) ? lhs : rhs; + const auto& max_size_set = (lhs.size() >= rhs.size()) ? lhs : rhs; + for (auto&& val : min_size_set) + if (max_size_set.find(val) != max_size_set.end()) + result.insert(val); + return result; +} + +template +static bool intersects(const Set& lhs, const Set& rhs) { + const auto& min_size_set = (lhs.size() < rhs.size()) ? lhs : rhs; + const auto& max_size_set = (lhs.size() >= rhs.size()) ? lhs : rhs; + for (auto&& val : min_size_set) + if (max_size_set.find(val) != max_size_set.end()) + return true; + return false; +} +} // namespace + +std::shared_ptr ov::hetero::SubgraphCollector::input_node( + const ov::hetero::SubgraphCollector::Input& input) const { + return input.get_source_output().get_node_shared_ptr(); +} + +ov::hetero::SubgraphCollector::SubgraphCollector(const std::shared_ptr& model, + const AffinitiesMap& affinities) + : _ordered_ops{model->get_ordered_ops()}, + _affinities{affinities}, + _graph_input_nodes{}, + _node_input_dependencies{}, + _subgraph_inputs{}, + _subgraph_parameter_to_prev_result{} { + init(); + split_cyclic_dependencies(); + _subgraph_ids = collect_subgraphs_ids(); +} + +void ov::hetero::SubgraphCollector::init() { + // Get all subgraph inputs using just node affinities. Also collect transitive closure + for (const auto& node : _ordered_ops) { + if (ov::op::util::is_parameter(node) || ov::op::util::is_constant(node)) { + _graph_input_nodes.insert(node); + _subgraph_inputs.insert(Input{node.get(), 0}); + _node_input_dependencies[node].insert(Input{node.get(), 0}); + } else { + auto inputs = node->inputs(); + auto& node_input_dependency = _node_input_dependencies[node]; + for (const auto& input : inputs) { + node_input_dependency.insert(input); + auto& inputDependency = _node_input_dependencies[input_node(input)]; + node_input_dependency.insert(inputDependency.begin(), inputDependency.end()); + if (_affinities.at(node) != _affinities.at(input_node(input))) { + _subgraph_inputs.insert(input); + } + } + } + } +} +void ov::hetero::SubgraphCollector::split_cyclic_dependencies() { + // Split cyclic dependencies. + for (size_t prev_subgraphs = 0, cyclic_split_step = 0; prev_subgraphs != _subgraph_inputs.size(); + ++cyclic_split_step) { + OPENVINO_ASSERT(cyclic_split_step < _ordered_ops.size(), "Cannot resolve cycles during submodels split!"); + prev_subgraphs = _subgraph_inputs.size(); + auto subgraph_ids = collect_subgraphs_ids(); + // All inputs that belong to the same subgraph as node + std::unordered_map, InputSet> node_subgraph_input_dependencies; + // All inputs that depends on the same subgraph as node + std::unordered_map, InputSet> node_subgraph_cyclic_iput_dependencies; + for (const auto& node : _ordered_ops) { + auto& node_subgraph_input_dependency = node_subgraph_input_dependencies[node]; + auto all_node_subgraph_inputs = intersection(_node_input_dependencies[node], _subgraph_inputs); + for (const auto& subgraphInput : all_node_subgraph_inputs) { + if (subgraph_ids[node] == subgraph_ids[subgraphInput.get_node()->shared_from_this()]) { + node_subgraph_input_dependency.emplace(subgraphInput); + } + } + auto& node_subgraph_cyclic_input_dependency = node_subgraph_cyclic_iput_dependencies[node]; + for (const auto& subgraphInput : all_node_subgraph_inputs) { + if (!ov::op::util::is_parameter(subgraphInput.get_node()) && + !ov::op::util::is_constant(subgraphInput.get_node()) && + subgraph_ids[node] == subgraph_ids[input_node(subgraphInput)]) { + node_subgraph_cyclic_input_dependency.emplace(subgraphInput); + } + } + } + + for (const auto& node : _ordered_ops) { + auto& node_subgraph_cyclic_input_dependency = node_subgraph_cyclic_iput_dependencies[node]; + if (!node_subgraph_cyclic_input_dependency.empty()) { + // Collect all subgraph inputs that cyclic subgraph output depends on + InputSet cyclicInputsDependencies; + for (const auto& cyclicInput : node_subgraph_cyclic_input_dependency) { + for (const auto& input : node_subgraph_input_dependencies[input_node(cyclicInput)]) { + cyclicInputsDependencies.emplace(input); + } + } + for (const auto& input : node->inputs()) { + auto& input_node_subgraph_cyclic_input_dependency = + node_subgraph_cyclic_iput_dependencies[input_node(input)]; + auto& input_node_subgraph_input_dependency = node_subgraph_input_dependencies[input_node(input)]; + if (!intersects(node_subgraph_cyclic_input_dependency, + input_node_subgraph_cyclic_input_dependency) && + intersects(cyclicInputsDependencies, input_node_subgraph_input_dependency)) { + _subgraph_inputs.insert(input); + } + } + } + } + } +} + +ov::hetero::SubgraphCollector::SubgraphIdsMap ov::hetero::SubgraphCollector::collect_subgraphs_ids() { + std::deque subgraph_ids; + NodeMap subgraph_id_ptrs; + for (const auto& node : _ordered_ops) { + auto all_node_inputs = node->inputs(); + std::vector inputs; + for (const auto& input : all_node_inputs) { + if (_subgraph_inputs.find(input) == _subgraph_inputs.end()) { + inputs.emplace_back(std::move(input)); + } + } + if (inputs.empty()) { + subgraph_ids.push_back(static_cast(subgraph_ids.size())); + subgraph_id_ptrs.emplace(node, &(subgraph_ids.back())); + } else { + auto first_input_subgraph_id_ptr = subgraph_id_ptrs[input_node(inputs.front())]; + for (const auto& input : inputs) { + auto input_id = *subgraph_id_ptrs[input_node(input)]; + for (auto& subgraph_id : subgraph_ids) { + if (subgraph_id == input_id) { + subgraph_id = *first_input_subgraph_id_ptr; + } + } + } + subgraph_id_ptrs.emplace(node, first_input_subgraph_id_ptr); + } + } + SubgraphIdsMap result; + for (const auto& subgraph_id_ptr : subgraph_id_ptrs) { + result.emplace(subgraph_id_ptr.first, *(subgraph_id_ptr.second)); + } + return result; +} + +void ov::hetero::SubgraphCollector::split_subgraphs_by_parameter_results() { + // Break graph using insertion of result parameter split + std::vector> results; + { + std::set> subgraph_outputs; + for (const auto& input : _subgraph_inputs) { + if (!ov::op::util::is_parameter(input.get_node()) && !ov::op::util::is_constant(input.get_node())) { + auto input_source_output = input.get_source_output(); + if (!ov::op::util::is_parameter(input_source_output.get_node()) && + !ov::op::util::is_constant(input_source_output.get_node())) { + subgraph_outputs.insert(input_source_output); + } + } + } + for (const auto& output : subgraph_outputs) { + auto output_subgraph_id = _subgraph_ids.at(output.get_node_shared_ptr()); + auto inputs = output.get_target_inputs(); + // Collect input subsets from other subgraphs. Each subset of inputs belongs to the same subgraph + std::map>> input_subsets; + for (const auto& input : inputs) { + auto input_subgraph_id = _subgraph_ids.at(input.get_node()->shared_from_this()); + if (output_subgraph_id != input_subgraph_id) { + input_subsets[input_subgraph_id].emplace(input); + } + } + // Avoid duplicate results on the same output port + auto result = std::make_shared(output); + const auto name = output.get_node()->get_friendly_name() + "_" + std::to_string(output.get_index()); + result->set_friendly_name(name + "_result"); + ov::copy_runtime_info(output.get_node_shared_ptr(), result); + _subgraph_ids.emplace(result, output_subgraph_id); + results.push_back(result); + for (const auto& input_subset : input_subsets) { + // Avoid duplicate parameters in the same subgraph + auto parameter = + std::make_shared(output.get_element_type(), output.get_partial_shape()); + parameter->set_friendly_name(name + "_parameter"); + for (const auto& input : input_subset.second) { + output.remove_target_input(input); + ov::copy_runtime_info(input.get_node()->shared_from_this(), parameter); + input.replace_source_output(parameter->output(0)); + _subgraph_ids.emplace(parameter, input_subset.first); + _subgraph_parameter_to_prev_result.emplace(parameter, result); + } + } + } + } +} + +std::vector ov::hetero::SubgraphCollector::get_ordered_subgraphs() { + // Break graph using insertion of result parameter split + split_subgraphs_by_parameter_results(); + + // Extracts subgraph parameters, results and affinities + auto subgraphs = collect_subgraphs(); + + // Subgraph topological sort + std::vector all_subgraphs; + for (const auto& subgraph : subgraphs) { + all_subgraphs.emplace_back(std::move(subgraph.second)); + } + + std::vector ordered_subgraphs; + NodeSet prev_results; + size_t subgraph_topo_sorts_step = 0; + do { + OPENVINO_ASSERT(subgraph_topo_sorts_step < subgraphs.size(), "Cannot sort subgraphs!"); + ++subgraph_topo_sorts_step; + std::vector new_ordered_subgraphs; + auto is_ordered_subgraph = [&](const Subgraph& subgraph) { + auto& parameters = subgraph._parameters; + return std::all_of( + parameters.begin(), + parameters.end(), + [&](const ov::ParameterVector::value_type& parameter) { + return (_graph_input_nodes.find(parameter) != _graph_input_nodes.end()) || + (prev_results.find(_subgraph_parameter_to_prev_result[parameter]) != prev_results.end()); + }); + }; + std::remove_copy_if(std::begin(all_subgraphs), + std::end(all_subgraphs), + std::back_inserter(new_ordered_subgraphs), + [&](const Subgraph& subgraph) { + return !is_ordered_subgraph(subgraph); + }); + all_subgraphs.erase(std::remove_if(std::begin(all_subgraphs), std::end(all_subgraphs), is_ordered_subgraph), + std::end(all_subgraphs)); + for (const auto& subgraph : new_ordered_subgraphs) { + for (const auto& result : subgraph._results) { + prev_results.insert(result); + } + } + std::move(std::begin(new_ordered_subgraphs), + std::end(new_ordered_subgraphs), + std::back_inserter(ordered_subgraphs)); + } while (!all_subgraphs.empty()); + return ordered_subgraphs; +} + +std::unordered_map +ov::hetero::SubgraphCollector::collect_subgraphs() { + std::unordered_map subgraphs; + // Extracts subgraph parameters, results and affinities + for (const auto& subgraph_id_ptr_value : _subgraph_ids) { + auto node = subgraph_id_ptr_value.first; + auto& subgraph = subgraphs[subgraph_id_ptr_value.second]; + if (ov::op::util::is_output(node)) { + subgraph._results.emplace_back(ov::as_type_ptr(node->shared_from_this())); + } else if (ov::op::util::is_parameter(node)) { + subgraph._parameters.emplace_back(ov::as_type_ptr(node->shared_from_this())); + } else if (ov::op::util::is_sink(node)) { + subgraph._sinks.emplace_back(ov::as_type_ptr(node->shared_from_this())); + } + auto it_affinity = _affinities.find(node); + if (it_affinity != _affinities.end()) { + subgraph._affinity = it_affinity->second; + } + } + return subgraphs; +} diff --git a/src/plugins/hetero/src/subgraph_collector.hpp b/src/plugins/hetero/src/subgraph_collector.hpp new file mode 100644 index 00000000000..cf04a73b078 --- /dev/null +++ b/src/plugins/hetero/src/subgraph_collector.hpp @@ -0,0 +1,61 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "openvino/runtime/common.hpp" +#include "openvino/runtime/core.hpp" + +namespace ov { +namespace hetero { + +class SubgraphCollector { +public: + struct Subgraph { + ov::ResultVector _results; + ov::ParameterVector _parameters; + ov::SinkVector _sinks; + std::string _affinity; + }; + template + using NodeMap = std::unordered_map, T>; + using NodeSet = std::unordered_set>; + using AffinitiesMap = NodeMap; + using SubgraphId = int; + using SubgraphIdsMap = NodeMap; + using ParameterResultMap = NodeMap>; + using Input = ov::Input; + using InputSet = std::set; + + SubgraphCollector(const std::shared_ptr& model, const AffinitiesMap& affinities); + SubgraphIdsMap get_subgraph_ids() { + return _subgraph_ids; + } + ParameterResultMap get_subgraph_parameter_to_prev_result() { + return _subgraph_parameter_to_prev_result; + } + std::vector get_ordered_subgraphs(); + +private: + void init(); + void split_cyclic_dependencies(); + void split_subgraphs_by_parameter_results(); + SubgraphIdsMap collect_subgraphs_ids(); + std::unordered_map collect_subgraphs(); + std::shared_ptr input_node(const Input& input) const; + + ov::NodeVector _ordered_ops; + AffinitiesMap _affinities; + NodeSet _graph_input_nodes; + NodeMap _node_input_dependencies; + InputSet _subgraph_inputs; + SubgraphIdsMap _subgraph_ids; + ParameterResultMap _subgraph_parameter_to_prev_result; +}; + +} // namespace hetero +} // namespace ov diff --git a/src/plugins/hetero/src/version.cpp b/src/plugins/hetero/src/version.cpp new file mode 100644 index 00000000000..290f818efab --- /dev/null +++ b/src/plugins/hetero/src/version.cpp @@ -0,0 +1,8 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "plugin.hpp" + +static const ov::Version version = {CI_BUILD_NUMBER, "openvino_hetero_plugin"}; +OV_DEFINE_PLUGIN_CREATE_FUNCTION(ov::hetero::Plugin, version) \ No newline at end of file diff --git a/src/plugins/hetero/tests/unit/CMakeLists.txt b/src/plugins/hetero/tests/unit/CMakeLists.txt new file mode 100644 index 00000000000..42056e8a0cd --- /dev/null +++ b/src/plugins/hetero/tests/unit/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME ov_hetero_unit_tests) + +if(BUILD_SHARED_LIBS) + set (OBJ_LIB $) +endif() + +ov_add_test_target( + NAME + ${TARGET_NAME} + ROOT + ${CMAKE_CURRENT_SOURCE_DIR} + INCLUDES + PUBLIC + $/src + ${CMAKE_CURRENT_SOURCE_DIR} + OBJECT_FILES + ${OBJ_LIB} + LINK_LIBRARIES + unit_test_utils + ngraphFunctions + DEPENDENCIES + mock_engine + ngraphFunctions + ADD_CLANG_FORMAT + LABELS + HETERO +) diff --git a/src/plugins/hetero/tests/unit/subgraph_collector.cpp b/src/plugins/hetero/tests/unit/subgraph_collector.cpp new file mode 100644 index 00000000000..1a710a4cc53 --- /dev/null +++ b/src/plugins/hetero/tests/unit/subgraph_collector.cpp @@ -0,0 +1,143 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "subgraph_collector.hpp" + +#include + +#include "common_test_utils/graph_comparator.hpp" +#include "openvino/core/except.hpp" +#include "openvino/op/ops.hpp" + +using namespace ov::hetero; + +namespace { +std::shared_ptr create_test_model() { + auto param = std::make_shared(ov::element::i64, ov::PartialShape{1, 3, 2, 2}); + param->set_friendly_name("input"); + auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); + const_value->set_friendly_name("const_val"); + auto add = std::make_shared(param, const_value); + add->set_friendly_name("add"); + auto subtract = std::make_shared(add, const_value); + subtract->set_friendly_name("sub"); + auto reshape_val = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + reshape_val->set_friendly_name("reshape_val"); + auto reshape = std::make_shared(subtract, reshape_val, true); + reshape->set_friendly_name("reshape"); + auto result = std::make_shared(reshape); + result->set_friendly_name("res"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} +std::shared_ptr create_subgraph_add() { + auto param = std::make_shared(ov::element::i64, ov::PartialShape{1, 3, 2, 2}); + param->set_friendly_name("input"); + auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); + const_value->set_friendly_name("const_val"); + auto add = std::make_shared(param, const_value); + add->set_friendly_name("add"); + auto result = std::make_shared(add); + result->set_friendly_name("add_0_result"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} +std::shared_ptr create_subgraph_sub() { + auto param = std::make_shared(ov::element::i64, ov::PartialShape{1, 3, 2, 2}); + param->set_friendly_name("add_0_parameter"); + auto const_value = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); + const_value->set_friendly_name("const_val"); + auto sub = std::make_shared(param, const_value); + sub->set_friendly_name("sub"); + auto result = std::make_shared(sub); + result->set_friendly_name("sub_0_result"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} +std::shared_ptr create_subgraph_reshape() { + auto param = std::make_shared(ov::element::i64, ov::PartialShape{1, 3, 2, 2}); + param->set_friendly_name("sub_0_parameter"); + auto reshape_val = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + reshape_val->set_friendly_name("reshape_val"); + auto reshape = std::make_shared(param, reshape_val, true); + reshape->set_friendly_name("reshape"); + auto result = std::make_shared(reshape); + result->set_friendly_name("res"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} +} // namespace + +class SubgraphCollectorTest : public testing::Test { + void SetUp() override { + m_model = create_test_model(); + m_submodels = {}; + m_submodels.push_back(create_subgraph_add()); + m_submodels.push_back(create_subgraph_sub()); + m_submodels.push_back(create_subgraph_reshape()); + } + + void TearDown() override {} + +protected: + std::shared_ptr m_model; + std::vector> m_submodels; +}; + +TEST_F(SubgraphCollectorTest, check_subgraphs) { + const std::map supported_ops = { + {"input", "MOCK.0"}, + {"const_val", "MOCK.0"}, + {"add", "MOCK.0"}, + {"sub", "MOCK.1"}, + {"reshape_val", "MOCK.0"}, + {"reshape", "MOCK.0"}, + {"res", "MOCK.0"}, + }; + const std::map expected_ids = { + {"input", 0}, + {"const_val", 0}, + {"add", 0}, + {"sub", 2}, + {"reshape_val", 3}, + {"reshape", 3}, + {"res", 3}, + }; + const std::map expected_parameter_to_prev_result_names = { + {"sub_0_parameter", "sub_0_result"}, + {"add_0_parameter", "add_0_result"}, + }; + SubgraphCollector::AffinitiesMap affinities; + SubgraphCollector::SubgraphIdsMap exptected_subgraphs_ids; + const auto ordered_ops = m_model->get_ordered_ops(); + for (const auto& node : ordered_ops) { + const auto name = node->get_friendly_name(); + OPENVINO_ASSERT(supported_ops.count(name)); + affinities[node] = supported_ops.at(name); + OPENVINO_ASSERT(expected_ids.count(name)); + exptected_subgraphs_ids[node] = expected_ids.at(name); + } + SubgraphCollector subgraph_collector(m_model, affinities); + auto actual_subgraphs_ids = subgraph_collector.get_subgraph_ids(); + for (auto& op : ordered_ops) { + ASSERT_EQ(actual_subgraphs_ids.at(op), exptected_subgraphs_ids.at(op)); + } + // Subgraphs are not collected yet + ASSERT_EQ(0, subgraph_collector.get_subgraph_parameter_to_prev_result().size()); + // Collect subgraphs + auto actual_subgraphs = subgraph_collector.get_ordered_subgraphs(); + const size_t submodels_number = 3; + ASSERT_EQ(submodels_number, actual_subgraphs.size()); + auto get_submodel = [&](size_t i) { + return std::make_shared(actual_subgraphs.at(i)._results, actual_subgraphs.at(i)._parameters); + }; + std::pair res; + for (size_t i = 0; i < submodels_number; i++) { + auto submodel = get_submodel(i); + auto res = compare_functions(m_submodels.at(i), submodel); + ASSERT_TRUE(res.first) << res.second; + } + auto actual_parameter_to_prev_result = subgraph_collector.get_subgraph_parameter_to_prev_result(); + ASSERT_EQ(actual_parameter_to_prev_result.size(), expected_parameter_to_prev_result_names.size()); + for (auto& item : actual_parameter_to_prev_result) { + ASSERT_EQ(item.second->get_friendly_name(), + expected_parameter_to_prev_result_names.at(item.first->get_friendly_name())); + } +}