diff --git a/src/common/transformations/include/transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp b/src/common/transformations/include/transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp new file mode 100644 index 00000000000..aa5409acad6 --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp @@ -0,0 +1,32 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace ov { +namespace pass { + +/** + * @ingroup ie_transformation_common_api + * @brief ConvolutionToGroupConvolutionFusion transformation replaces following graph: + * Split (or VariadicSplit) + * / \ + * Conv ... Conv + * \ / + * \ / + * Concat + * + * to GroupConvolution + */ +class TRANSFORMATIONS_API ConvolutionToGroupConvolutionFusion : public MatcherPass { +public: + OPENVINO_RTTI("ConvolutionToGroupConvolutionFusion", "0"); + ConvolutionToGroupConvolutionFusion(); +}; + +} // namespace pass +} // namespace ov diff --git a/src/common/transformations/src/transformations/common_optimizations/convolution_to_group_convolution.cpp b/src/common/transformations/src/transformations/common_optimizations/convolution_to_group_convolution.cpp new file mode 100644 index 00000000000..835d62cc56f --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/convolution_to_group_convolution.cpp @@ -0,0 +1,164 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include + +#include "itt.hpp" +#include "transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp" + +static bool compare_convolutions(const ov::opset10::Convolution* conv1, ov::Node* node) { + const auto conv2 = ov::as_type(node); + if (!conv2) + return false; + return conv1->get_strides() == conv2->get_strides() && conv1->get_pads_begin() == conv2->get_pads_begin() && + conv1->get_pads_end() == conv2->get_pads_end() && conv1->get_dilations() == conv2->get_dilations() && + conv1->get_auto_pad() == conv2->get_auto_pad(); +} + +static int64_t get_split_axis(const std::shared_ptr& split) { + const auto axis = ov::as_type(split->get_input_node_ptr(1)); + if (!axis) + return -1; + auto axis_value = axis->cast_vector()[0]; + if (axis_value < 0) { + const auto& input_rank = split->get_input_partial_shape(0).rank(); + if (input_rank.is_dynamic()) + return -1; + axis_value += input_rank.get_length(); + } + + return axis_value; +} + +static std::shared_ptr create_new_weights(ov::pass::NodeRegistry& node_registry, + const std::shared_ptr& concat) { + const auto concat_input = concat->get_input_node_ptr(0); + if (concat_input->get_input_partial_shape(1).is_dynamic()) + return nullptr; + + // unsqueeze weights shape from (O, I, X, Y) to (1, O, I, X, Y) + const auto& weights_shape = concat_input->get_input_shape(1); + ov::Shape new_shape = weights_shape; + new_shape.insert(new_shape.begin(), 1); + + const size_t num_inputs = concat->get_input_size(); + ov::OutputVector weights_to_concat; + weights_to_concat.reserve(num_inputs); + + for (size_t i = 0; i < num_inputs; i++) { + const auto conv = concat->get_input_node_shared_ptr(i); + const auto weights = conv->get_input_node_shared_ptr(1); + const auto& shape = weights->get_output_partial_shape(0); + if (shape.is_dynamic() || weights->get_output_shape(0) != weights_shape) + return nullptr; + if (auto constant = ov::as_type_ptr(weights)) { + weights_to_concat.push_back(node_registry.make(*constant, new_shape)); + } else { + weights_to_concat.push_back(node_registry.make( + weights, + ov::opset10::Constant::create(ov::element::i32, ov::Shape{}, {0}))); + } + weights_to_concat.back().get_node()->set_friendly_name(weights->get_friendly_name()); + } + + return node_registry.make(weights_to_concat, 0); +} + +ov::pass::ConvolutionToGroupConvolutionFusion::ConvolutionToGroupConvolutionFusion() { + MATCHER_SCOPE(ConvolutionToGroupConvolutionFusion); + + auto has_conv_inputs = [](const Output& node) -> bool { + const auto concat = node.get_node(); + size_t num_inputs = concat->get_input_size(); + if (num_inputs == 0) + return false; + + const auto first_conv = as_type(concat->get_input_node_ptr(0)); + if (!first_conv) + return false; + + const auto split = first_conv->get_input_node_ptr(0); + if (!is_type(split) && !is_type(split)) + return false; + + // go through Concat inputs and check + // - if all of them are Convolutions + // - if those Convolutions have the same Split input + for (size_t i = 1; i < concat->get_input_size(); i++) { + const auto conv = concat->get_input_node_ptr(i); + if (conv->get_input_node_ptr(0) != split) + return false; + if (!compare_convolutions(first_conv, conv)) + return false; + } + return true; + }; + auto concat_label = pattern::wrap_type(has_conv_inputs); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& pattern_value_map = m.get_pattern_value_map(); + const auto& concat = pattern_value_map.at(concat_label).get_node_shared_ptr(); + + const auto first_conv = as_type_ptr(concat->get_input_node_shared_ptr(0)); + const auto split = first_conv->get_input_node_shared_ptr(0); + const bool is_split = is_type(split); + const bool is_variadic_split = is_type(split); + if (!is_split && !is_variadic_split) + return false; + + if (get_split_axis(split) != 1) + return false; + + if (is_variadic_split) { + // split_lengths in VariadicSplit must have the same values + if (auto split_lengths = as_type(split->get_input_node_ptr(1))) { + const auto split_lengths_values = split_lengths->cast_vector(); + const auto first_length = split_lengths_values[0]; + if (!std::all_of(split_lengths_values.begin() + 1, + split_lengths_values.end(), + [first_length](int split_length) { + return split_length == first_length; + })) + return false; + } else { + return false; + } + } + + NodeRegistry node_registry; + const auto weights = create_new_weights(node_registry, concat); + if (!weights) + return false; + + const auto conv = node_registry.make(split->get_input_node_shared_ptr(0), + weights, + first_conv->get_strides(), + first_conv->get_pads_begin(), + first_conv->get_pads_end(), + first_conv->get_dilations(), + first_conv->get_auto_pad()); + conv->set_friendly_name(concat->get_friendly_name()); + register_new_node(conv); + + const size_t concat_num_inputs = concat->get_input_size(); + NodeVector from; + from.reserve(concat_num_inputs + 2); + from.push_back(split); + from.push_back(first_conv); + for (size_t i = 1; i < concat_num_inputs; i++) { + from.push_back(concat->get_input_node_shared_ptr(i)); + } + from.push_back(concat); + + copy_runtime_info(from, node_registry.get()); + replace_node(concat, conv); + + return true; + }; + + auto m = std::make_shared(concat_label, matcher_name); + this->register_matcher(m, callback); +} diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index 21c3e3a400f..20f2ede4454 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -189,6 +190,7 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr + +#include +#include +#include + +#include "common_test_utils/ngraph_test_utils.hpp" + +using namespace testing; +using namespace ov; + +TEST_F(TransformationTestsF, ConvToGroupConvFusionSplit) { + Shape input_shape{2, 10, 14, 14}; + size_t num_splits = 5; + int axis = 1; + Shape weights_shape{3, input_shape[1] / num_splits, 1, 1}; + const auto spatial_dim_size = weights_shape.size() - 2; + Strides strides(spatial_dim_size, 1); + CoordinateDiff pads_begin(spatial_dim_size, 0); + CoordinateDiff pads_end(spatial_dim_size, 0); + Strides dilations(spatial_dim_size, 1); + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split = std::make_shared(data, axis_node, num_splits); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + for (size_t i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1}); + concat_inputs.push_back(std::make_shared(split->output(i), + weights, + strides, + pads_begin, + pads_end, + dilations)); + } + const auto concat = std::make_shared(concat_inputs, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + { + const auto data = std::make_shared(element::f32, input_shape); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + Shape new_weights_shape = weights_shape; + new_weights_shape.insert(new_weights_shape.begin(), 1); + for (size_t i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, new_weights_shape, {i + 1}); + concat_inputs.push_back(weights); + } + const auto concat = std::make_shared(concat_inputs, 0); + const auto conv = + std::make_shared(data, concat, strides, pads_begin, pads_end, dilations); + function_ref = std::make_shared(conv, ParameterVector{data}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + +TEST_F(TransformationTestsF, ConvToGroupConvFusionVariadicSplit) { + Shape input_shape{2, 10, 14, 14}; + size_t num_splits = 5; + int axis = 1; + Shape weights_shape{3, input_shape[1] / num_splits, 1, 1}; + const auto spatial_dim_size = weights_shape.size() - 2; + Strides strides(spatial_dim_size, 1); + CoordinateDiff pads_begin(spatial_dim_size, 0); + CoordinateDiff pads_end(spatial_dim_size, 0); + Strides dilations(spatial_dim_size, 1); + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split_lengths = + opset10::Constant::create(element::i32, Shape{num_splits}, std::vector(num_splits, 2)); + const auto split = std::make_shared(data, axis_node, split_lengths); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + for (size_t i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1}); + concat_inputs.push_back(std::make_shared(split->output(i), + weights, + strides, + pads_begin, + pads_end, + dilations)); + } + const auto concat = std::make_shared(concat_inputs, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + { + const auto data = std::make_shared(element::f32, input_shape); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + Shape new_weights_shape = weights_shape; + new_weights_shape.insert(new_weights_shape.begin(), 1); + for (size_t i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, new_weights_shape, {i + 1}); + concat_inputs.push_back(weights); + } + const auto concat = std::make_shared(concat_inputs, 0); + const auto conv = + std::make_shared(data, concat, strides, pads_begin, pads_end, dilations); + function_ref = std::make_shared(conv, ParameterVector{data}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + +TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionSplitInvalidAxis) { + Shape input_shape{2, 10, 14, 14}; + int num_splits = 2; + int axis = 2; + Shape weights_shape{3, input_shape[1], 1, 1}; + const auto spatial_dim_size = weights_shape.size() - 2; + Strides strides(spatial_dim_size, 1); + CoordinateDiff pads_begin(spatial_dim_size, 0); + CoordinateDiff pads_end(spatial_dim_size, 0); + Strides dilations(spatial_dim_size, 1); + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split = std::make_shared(data, axis_node, num_splits); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + for (int i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1}); + concat_inputs.push_back(std::make_shared(split->output(i), + weights, + strides, + pads_begin, + pads_end, + dilations)); + } + const auto concat = std::make_shared(concat_inputs, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + +TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionSplitNotMatchingConvAttributes) { + Shape input_shape{2, 10, 14, 14}; + size_t num_splits = 2; + int axis = 1; + const auto spatial_dim_size = 2; + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split = std::make_shared(data, axis_node, num_splits); + + const auto weights1 = opset10::Constant::create(element::f32, Shape{3, input_shape[1] / num_splits, 2, 2}, {1}); + Strides strides1(spatial_dim_size, 1); + CoordinateDiff pads_begin1(spatial_dim_size, 1); + CoordinateDiff pads_end1(spatial_dim_size, 1); + Strides dilations1(spatial_dim_size, 1); + const auto conv1 = std::make_shared(split->output(0), + weights1, + strides1, + pads_begin1, + pads_end1, + dilations1); + + const auto weights2 = opset10::Constant::create(element::f32, Shape{3, input_shape[1] / num_splits, 4, 4}, {1}); + Strides strides2(spatial_dim_size, 1); + CoordinateDiff pads_begin2(spatial_dim_size, 2); + CoordinateDiff pads_end2(spatial_dim_size, 2); + Strides dilations2(spatial_dim_size, 1); + const auto conv2 = std::make_shared(split->output(1), + weights2, + strides2, + pads_begin2, + pads_end2, + dilations2); + const auto concat = std::make_shared(OutputVector{conv1, conv2}, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + +TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionVariadicSplitUnevenSplitLengths) { + Shape input_shape{2, 10, 14, 14}; + int axis = 1; + const auto spatial_dim_size = 2; + Strides strides(spatial_dim_size, 1); + CoordinateDiff pads_begin(spatial_dim_size, 0); + CoordinateDiff pads_end(spatial_dim_size, 0); + Strides dilations(spatial_dim_size, 1); + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split_lengths = opset10::Constant::create(element::i32, + Shape{2}, + std::vector{3, static_cast(input_shape[1]) - 3}); + const auto split = std::make_shared(data, axis_node, split_lengths); + const auto weights1 = opset10::Constant::create(element::f32, Shape{3, 3, 1, 1}, {1}); + const auto conv1 = std::make_shared(split->output(0), + weights1, + strides, + pads_begin, + pads_end, + dilations); + const auto weights2 = opset10::Constant::create(element::f32, Shape{3, 7, 1, 1}, {2}); + const auto conv2 = std::make_shared(split->output(1), + weights2, + strides, + pads_begin, + pads_end, + dilations); + const auto concat = std::make_shared(OutputVector{conv1, conv2}, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +}