Add ConvolutionToGroupConvolutionFusion (#16688)

Fuses Split->series of Conv->Concat to GroupConvolution op.

Ticket: 105170

src/common/transformations/include/transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp

@@ -0,0 +1,32 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <openvino/pass/graph_rewrite.hpp>
+#include <transformations_visibility.hpp>
+
+namespace ov {
+namespace pass {
+
+/**
+ * @ingroup ie_transformation_common_api
+ * @brief ConvolutionToGroupConvolutionFusion transformation replaces following graph:
+ *                    Split (or VariadicSplit)
+ *                  /       \
+ *                Conv ... Conv
+ *                  \       /
+ *                   \     /
+ *                    Concat
+ *
+ * to GroupConvolution
+ */
+class TRANSFORMATIONS_API ConvolutionToGroupConvolutionFusion : public MatcherPass {
+public:
+    OPENVINO_RTTI("ConvolutionToGroupConvolutionFusion", "0");
+    ConvolutionToGroupConvolutionFusion();
+};
+
+}  // namespace pass
+}  // namespace ov

src/common/transformations/src/transformations/common_optimizations/convolution_to_group_convolution.cpp

@@ -0,0 +1,164 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <openvino/core/rt_info.hpp>
+#include <openvino/opsets/opset10.hpp>
+#include <openvino/pass/pattern/op/wrap_type.hpp>
+
+#include "itt.hpp"
+#include "transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp"
+
+static bool compare_convolutions(const ov::opset10::Convolution* conv1, ov::Node* node) {
+    const auto conv2 = ov::as_type<ov::opset10::Convolution>(node);
+    if (!conv2)
+        return false;
+    return conv1->get_strides() == conv2->get_strides() && conv1->get_pads_begin() == conv2->get_pads_begin() &&
+           conv1->get_pads_end() == conv2->get_pads_end() && conv1->get_dilations() == conv2->get_dilations() &&
+           conv1->get_auto_pad() == conv2->get_auto_pad();
+}
+
+static int64_t get_split_axis(const std::shared_ptr<ov::Node>& split) {
+    const auto axis = ov::as_type<ov::opset10::Constant>(split->get_input_node_ptr(1));
+    if (!axis)
+        return -1;
+    auto axis_value = axis->cast_vector<int64_t>()[0];
+    if (axis_value < 0) {
+        const auto& input_rank = split->get_input_partial_shape(0).rank();
+        if (input_rank.is_dynamic())
+            return -1;
+        axis_value += input_rank.get_length();
+    }
+
+    return axis_value;
+}
+
+static std::shared_ptr<ov::opset10::Concat> create_new_weights(ov::pass::NodeRegistry& node_registry,
+                                                               const std::shared_ptr<ov::Node>& concat) {
+    const auto concat_input = concat->get_input_node_ptr(0);
+    if (concat_input->get_input_partial_shape(1).is_dynamic())
+        return nullptr;
+
+    // unsqueeze weights shape from (O, I, X, Y) to (1, O, I, X, Y)
+    const auto& weights_shape = concat_input->get_input_shape(1);
+    ov::Shape new_shape = weights_shape;
+    new_shape.insert(new_shape.begin(), 1);
+
+    const size_t num_inputs = concat->get_input_size();
+    ov::OutputVector weights_to_concat;
+    weights_to_concat.reserve(num_inputs);
+
+    for (size_t i = 0; i < num_inputs; i++) {
+        const auto conv = concat->get_input_node_shared_ptr(i);
+        const auto weights = conv->get_input_node_shared_ptr(1);
+        const auto& shape = weights->get_output_partial_shape(0);
+        if (shape.is_dynamic() || weights->get_output_shape(0) != weights_shape)
+            return nullptr;
+        if (auto constant = ov::as_type_ptr<ov::opset10::Constant>(weights)) {
+            weights_to_concat.push_back(node_registry.make<ov::opset10::Constant>(*constant, new_shape));
+        } else {
+            weights_to_concat.push_back(node_registry.make<ov::opset10::Unsqueeze>(
+                weights,
+                ov::opset10::Constant::create(ov::element::i32, ov::Shape{}, {0})));
+        }
+        weights_to_concat.back().get_node()->set_friendly_name(weights->get_friendly_name());
+    }
+
+    return node_registry.make<ov::opset10::Concat>(weights_to_concat, 0);
+}
+
+ov::pass::ConvolutionToGroupConvolutionFusion::ConvolutionToGroupConvolutionFusion() {
+    MATCHER_SCOPE(ConvolutionToGroupConvolutionFusion);
+
+    auto has_conv_inputs = [](const Output<Node>& node) -> bool {
+        const auto concat = node.get_node();
+        size_t num_inputs = concat->get_input_size();
+        if (num_inputs == 0)
+            return false;
+
+        const auto first_conv = as_type<opset10::Convolution>(concat->get_input_node_ptr(0));
+        if (!first_conv)
+            return false;
+
+        const auto split = first_conv->get_input_node_ptr(0);
+        if (!is_type<opset10::Split>(split) && !is_type<opset10::VariadicSplit>(split))
+            return false;
+
+        // go through Concat inputs and check
+        // - if all of them are Convolutions
+        // - if those Convolutions have the same Split input
+        for (size_t i = 1; i < concat->get_input_size(); i++) {
+            const auto conv = concat->get_input_node_ptr(i);
+            if (conv->get_input_node_ptr(0) != split)
+                return false;
+            if (!compare_convolutions(first_conv, conv))
+                return false;
+        }
+        return true;
+    };
+    auto concat_label = pattern::wrap_type<opset10::Concat>(has_conv_inputs);
+
+    matcher_pass_callback callback = [=](pattern::Matcher& m) {
+        const auto& pattern_value_map = m.get_pattern_value_map();
+        const auto& concat = pattern_value_map.at(concat_label).get_node_shared_ptr();
+
+        const auto first_conv = as_type_ptr<opset10::Convolution>(concat->get_input_node_shared_ptr(0));
+        const auto split = first_conv->get_input_node_shared_ptr(0);
+        const bool is_split = is_type<opset10::Split>(split);
+        const bool is_variadic_split = is_type<opset10::VariadicSplit>(split);
+        if (!is_split && !is_variadic_split)
+            return false;
+
+        if (get_split_axis(split) != 1)
+            return false;
+
+        if (is_variadic_split) {
+            // split_lengths in VariadicSplit must have the same values
+            if (auto split_lengths = as_type<opset10::Constant>(split->get_input_node_ptr(1))) {
+                const auto split_lengths_values = split_lengths->cast_vector<int>();
+                const auto first_length = split_lengths_values[0];
+                if (!std::all_of(split_lengths_values.begin() + 1,
+                                 split_lengths_values.end(),
+                                 [first_length](int split_length) {
+                                     return split_length == first_length;
+                                 }))
+                    return false;
+            } else {
+                return false;
+            }
+        }
+
+        NodeRegistry node_registry;
+        const auto weights = create_new_weights(node_registry, concat);
+        if (!weights)
+            return false;
+
+        const auto conv = node_registry.make<opset10::GroupConvolution>(split->get_input_node_shared_ptr(0),
+                                                                        weights,
+                                                                        first_conv->get_strides(),
+                                                                        first_conv->get_pads_begin(),
+                                                                        first_conv->get_pads_end(),
+                                                                        first_conv->get_dilations(),
+                                                                        first_conv->get_auto_pad());
+        conv->set_friendly_name(concat->get_friendly_name());
+        register_new_node(conv);
+
+        const size_t concat_num_inputs = concat->get_input_size();
+        NodeVector from;
+        from.reserve(concat_num_inputs + 2);
+        from.push_back(split);
+        from.push_back(first_conv);
+        for (size_t i = 1; i < concat_num_inputs; i++) {
+            from.push_back(concat->get_input_node_shared_ptr(i));
+        }
+        from.push_back(concat);
+
+        copy_runtime_info(from, node_registry.get());
+        replace_node(concat, conv);
+
+        return true;
+    };
+
+    auto m = std::make_shared<pattern::Matcher>(concat_label, matcher_name);
+    this->register_matcher(m, callback);
+}

src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp

@@ -15,6 +15,7 @@
 #include <transformations/common_optimizations/conv_to_binary_conv.hpp>
 #include <transformations/common_optimizations/convert_nms_gather_path_to_unsigned.hpp>
 #include <transformations/common_optimizations/convert_quantize_dequantize.hpp>
+#include <transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp>
 #include <transformations/common_optimizations/depth_to_space_fusion.hpp>
 #include <transformations/common_optimizations/dilated_convolution_converter.hpp>
 #include <transformations/common_optimizations/disable_random_uniform_constant_folding.hpp>
@@ -189,6 +190,7 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr<ngraph::Fu
     ADD_MATCHER(common_fusions, RandomUniformFusion)
     ADD_MATCHER(common_fusions, ConvertTensorIteratorToSequence)
     ADD_MATCHER(common_fusions, SplitConcatPairToInterpolateFusion, m_use_shapes)
+    ADD_MATCHER(common_fusions, ConvolutionToGroupConvolutionFusion)
     if (m_use_shapes) {
         ADD_MATCHER(common_fusions, NearestNeighborUpsamplingFusion)
     }

src/common/transformations/tests/common_optimizations/convolution_to_group_convolution_fusion.cpp

@@ -0,0 +1,235 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <gtest/gtest.h>
+
+#include <openvino/opsets/opset10.hpp>
+#include <openvino/pass/manager.hpp>
+#include <transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp>
+
+#include "common_test_utils/ngraph_test_utils.hpp"
+
+using namespace testing;
+using namespace ov;
+
+TEST_F(TransformationTestsF, ConvToGroupConvFusionSplit) {
+    Shape input_shape{2, 10, 14, 14};
+    size_t num_splits = 5;
+    int axis = 1;
+    Shape weights_shape{3, input_shape[1] / num_splits, 1, 1};
+    const auto spatial_dim_size = weights_shape.size() - 2;
+    Strides strides(spatial_dim_size, 1);
+    CoordinateDiff pads_begin(spatial_dim_size, 0);
+    CoordinateDiff pads_end(spatial_dim_size, 0);
+    Strides dilations(spatial_dim_size, 1);
+
+    {
+        const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
+        const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
+        const auto split = std::make_shared<opset10::Split>(data, axis_node, num_splits);
+        OutputVector concat_inputs;
+        concat_inputs.reserve(num_splits);
+        for (size_t i = 0; i < num_splits; i++) {
+            const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1});
+            concat_inputs.push_back(std::make_shared<opset10::Convolution>(split->output(i),
+                                                                           weights,
+                                                                           strides,
+                                                                           pads_begin,
+                                                                           pads_end,
+                                                                           dilations));
+        }
+        const auto concat = std::make_shared<opset10::Concat>(concat_inputs, axis);
+        function = std::make_shared<Model>(concat, ParameterVector{data});
+        manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
+    }
+
+    {
+        const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
+        OutputVector concat_inputs;
+        concat_inputs.reserve(num_splits);
+        Shape new_weights_shape = weights_shape;
+        new_weights_shape.insert(new_weights_shape.begin(), 1);
+        for (size_t i = 0; i < num_splits; i++) {
+            const auto weights = opset10::Constant::create(element::f32, new_weights_shape, {i + 1});
+            concat_inputs.push_back(weights);
+        }
+        const auto concat = std::make_shared<opset10::Concat>(concat_inputs, 0);
+        const auto conv =
+            std::make_shared<opset10::GroupConvolution>(data, concat, strides, pads_begin, pads_end, dilations);
+        function_ref = std::make_shared<Model>(conv, ParameterVector{data});
+    }
+
+    comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
+    comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
+}
+
+TEST_F(TransformationTestsF, ConvToGroupConvFusionVariadicSplit) {
+    Shape input_shape{2, 10, 14, 14};
+    size_t num_splits = 5;
+    int axis = 1;
+    Shape weights_shape{3, input_shape[1] / num_splits, 1, 1};
+    const auto spatial_dim_size = weights_shape.size() - 2;
+    Strides strides(spatial_dim_size, 1);
+    CoordinateDiff pads_begin(spatial_dim_size, 0);
+    CoordinateDiff pads_end(spatial_dim_size, 0);
+    Strides dilations(spatial_dim_size, 1);
+
+    {
+        const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
+        const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
+        const auto split_lengths =
+            opset10::Constant::create(element::i32, Shape{num_splits}, std::vector<int>(num_splits, 2));
+        const auto split = std::make_shared<opset10::VariadicSplit>(data, axis_node, split_lengths);
+        OutputVector concat_inputs;
+        concat_inputs.reserve(num_splits);
+        for (size_t i = 0; i < num_splits; i++) {
+            const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1});
+            concat_inputs.push_back(std::make_shared<opset10::Convolution>(split->output(i),
+                                                                           weights,
+                                                                           strides,
+                                                                           pads_begin,
+                                                                           pads_end,
+                                                                           dilations));
+        }
+        const auto concat = std::make_shared<opset10::Concat>(concat_inputs, axis);
+        function = std::make_shared<Model>(concat, ParameterVector{data});
+        manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
+    }
+
+    {
+        const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
+        OutputVector concat_inputs;
+        concat_inputs.reserve(num_splits);
+        Shape new_weights_shape = weights_shape;
+        new_weights_shape.insert(new_weights_shape.begin(), 1);
+        for (size_t i = 0; i < num_splits; i++) {
+            const auto weights = opset10::Constant::create(element::f32, new_weights_shape, {i + 1});
+            concat_inputs.push_back(weights);
+        }
+        const auto concat = std::make_shared<opset10::Concat>(concat_inputs, 0);
+        const auto conv =
+            std::make_shared<opset10::GroupConvolution>(data, concat, strides, pads_begin, pads_end, dilations);
+        function_ref = std::make_shared<Model>(conv, ParameterVector{data});
+    }
+
+    comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
+    comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
+}
+
+TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionSplitInvalidAxis) {
+    Shape input_shape{2, 10, 14, 14};
+    int num_splits = 2;
+    int axis = 2;
+    Shape weights_shape{3, input_shape[1], 1, 1};
+    const auto spatial_dim_size = weights_shape.size() - 2;
+    Strides strides(spatial_dim_size, 1);
+    CoordinateDiff pads_begin(spatial_dim_size, 0);
+    CoordinateDiff pads_end(spatial_dim_size, 0);
+    Strides dilations(spatial_dim_size, 1);
+
+    {
+        const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
+        const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
+        const auto split = std::make_shared<opset10::Split>(data, axis_node, num_splits);
+        OutputVector concat_inputs;
+        concat_inputs.reserve(num_splits);
+        for (int i = 0; i < num_splits; i++) {
+            const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1});
+            concat_inputs.push_back(std::make_shared<opset10::Convolution>(split->output(i),
+                                                                           weights,
+                                                                           strides,
+                                                                           pads_begin,
+                                                                           pads_end,
+                                                                           dilations));
+        }
+        const auto concat = std::make_shared<opset10::Concat>(concat_inputs, axis);
+        function = std::make_shared<Model>(concat, ParameterVector{data});
+        manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
+    }
+
+    comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
+    comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
+}
+
+TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionSplitNotMatchingConvAttributes) {
+    Shape input_shape{2, 10, 14, 14};
+    size_t num_splits = 2;
+    int axis = 1;
+    const auto spatial_dim_size = 2;
+
+    {
+        const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
+        const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
+        const auto split = std::make_shared<opset10::Split>(data, axis_node, num_splits);
+
+        const auto weights1 = opset10::Constant::create(element::f32, Shape{3, input_shape[1] / num_splits, 2, 2}, {1});
+        Strides strides1(spatial_dim_size, 1);
+        CoordinateDiff pads_begin1(spatial_dim_size, 1);
+        CoordinateDiff pads_end1(spatial_dim_size, 1);
+        Strides dilations1(spatial_dim_size, 1);
+        const auto conv1 = std::make_shared<opset10::Convolution>(split->output(0),
+                                                                  weights1,
+                                                                  strides1,
+                                                                  pads_begin1,
+                                                                  pads_end1,
+                                                                  dilations1);
+
+        const auto weights2 = opset10::Constant::create(element::f32, Shape{3, input_shape[1] / num_splits, 4, 4}, {1});
+        Strides strides2(spatial_dim_size, 1);
+        CoordinateDiff pads_begin2(spatial_dim_size, 2);
+        CoordinateDiff pads_end2(spatial_dim_size, 2);
+        Strides dilations2(spatial_dim_size, 1);
+        const auto conv2 = std::make_shared<opset10::Convolution>(split->output(1),
+                                                                  weights2,
+                                                                  strides2,
+                                                                  pads_begin2,
+                                                                  pads_end2,
+                                                                  dilations2);
+        const auto concat = std::make_shared<opset10::Concat>(OutputVector{conv1, conv2}, axis);
+        function = std::make_shared<Model>(concat, ParameterVector{data});
+        manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
+    }
+
+    comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
+    comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
+}
+
+TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionVariadicSplitUnevenSplitLengths) {
+    Shape input_shape{2, 10, 14, 14};
+    int axis = 1;
+    const auto spatial_dim_size = 2;
+    Strides strides(spatial_dim_size, 1);
+    CoordinateDiff pads_begin(spatial_dim_size, 0);
+    CoordinateDiff pads_end(spatial_dim_size, 0);
+    Strides dilations(spatial_dim_size, 1);
+
+    {
+        const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
+        const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
+        const auto split_lengths = opset10::Constant::create(element::i32,
+                                                             Shape{2},
+                                                             std::vector<int>{3, static_cast<int>(input_shape[1]) - 3});
+        const auto split = std::make_shared<opset10::VariadicSplit>(data, axis_node, split_lengths);
+        const auto weights1 = opset10::Constant::create(element::f32, Shape{3, 3, 1, 1}, {1});
+        const auto conv1 = std::make_shared<opset10::Convolution>(split->output(0),
+                                                                  weights1,
+                                                                  strides,
+                                                                  pads_begin,
+                                                                  pads_end,
+                                                                  dilations);
+        const auto weights2 = opset10::Constant::create(element::f32, Shape{3, 7, 1, 1}, {2});
+        const auto conv2 = std::make_shared<opset10::Convolution>(split->output(1),
+                                                                  weights2,
+                                                                  strides,
+                                                                  pads_begin,
+                                                                  pads_end,
+                                                                  dilations);
+        const auto concat = std::make_shared<opset10::Concat>(OutputVector{conv1, conv2}, axis);
+        function = std::make_shared<Model>(concat, ParameterVector{data});
+        manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
+    }
+
+    comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
+    comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
+}