Add ConvolutionToGroupConvolutionFusion (#16688)
Fuses Split->series of Conv->Concat to GroupConvolution op. Ticket: 105170
This commit is contained in:
Mateusz Tabaka
GitHub
parent
6237868437
commit
03ab0e4388
@ -0,0 +1,32 @@
|
||||
// Copyright (C) 2023 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <openvino/pass/graph_rewrite.hpp>
|
||||
#include <transformations_visibility.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace pass {
|
||||
|
||||
/**
|
||||
* @ingroup ie_transformation_common_api
|
||||
* @brief ConvolutionToGroupConvolutionFusion transformation replaces following graph:
|
||||
* Split (or VariadicSplit)
|
||||
* / \
|
||||
* Conv ... Conv
|
||||
* \ /
|
||||
* \ /
|
||||
* Concat
|
||||
*
|
||||
* to GroupConvolution
|
||||
*/
|
||||
class TRANSFORMATIONS_API ConvolutionToGroupConvolutionFusion : public MatcherPass {
|
||||
public:
|
||||
OPENVINO_RTTI("ConvolutionToGroupConvolutionFusion", "0");
|
||||
ConvolutionToGroupConvolutionFusion();
|
||||
};
|
||||
|
||||
} // namespace pass
|
||||
} // namespace ov
|
||||
@ -0,0 +1,164 @@
|
||||
// Copyright (C) 2023 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include <openvino/core/rt_info.hpp>
|
||||
#include <openvino/opsets/opset10.hpp>
|
||||
#include <openvino/pass/pattern/op/wrap_type.hpp>
|
||||
|
||||
#include "itt.hpp"
|
||||
#include "transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp"
|
||||
|
||||
static bool compare_convolutions(const ov::opset10::Convolution* conv1, ov::Node* node) {
|
||||
const auto conv2 = ov::as_type<ov::opset10::Convolution>(node);
|
||||
if (!conv2)
|
||||
return false;
|
||||
return conv1->get_strides() == conv2->get_strides() && conv1->get_pads_begin() == conv2->get_pads_begin() &&
|
||||
conv1->get_pads_end() == conv2->get_pads_end() && conv1->get_dilations() == conv2->get_dilations() &&
|
||||
conv1->get_auto_pad() == conv2->get_auto_pad();
|
||||
}
|
||||
|
||||
static int64_t get_split_axis(const std::shared_ptr<ov::Node>& split) {
|
||||
const auto axis = ov::as_type<ov::opset10::Constant>(split->get_input_node_ptr(1));
|
||||
if (!axis)
|
||||
return -1;
|
||||
auto axis_value = axis->cast_vector<int64_t>()[0];
|
||||
if (axis_value < 0) {
|
||||
const auto& input_rank = split->get_input_partial_shape(0).rank();
|
||||
if (input_rank.is_dynamic())
|
||||
return -1;
|
||||
axis_value += input_rank.get_length();
|
||||
}
|
||||
|
||||
return axis_value;
|
||||
}
|
||||
|
||||
static std::shared_ptr<ov::opset10::Concat> create_new_weights(ov::pass::NodeRegistry& node_registry,
|
||||
const std::shared_ptr<ov::Node>& concat) {
|
||||
const auto concat_input = concat->get_input_node_ptr(0);
|
||||
if (concat_input->get_input_partial_shape(1).is_dynamic())
|
||||
return nullptr;
|
||||
|
||||
// unsqueeze weights shape from (O, I, X, Y) to (1, O, I, X, Y)
|
||||
const auto& weights_shape = concat_input->get_input_shape(1);
|
||||
ov::Shape new_shape = weights_shape;
|
||||
new_shape.insert(new_shape.begin(), 1);
|
||||
|
||||
const size_t num_inputs = concat->get_input_size();
|
||||
ov::OutputVector weights_to_concat;
|
||||
weights_to_concat.reserve(num_inputs);
|
||||
|
||||
for (size_t i = 0; i < num_inputs; i++) {
|
||||
const auto conv = concat->get_input_node_shared_ptr(i);
|
||||
const auto weights = conv->get_input_node_shared_ptr(1);
|
||||
const auto& shape = weights->get_output_partial_shape(0);
|
||||
if (shape.is_dynamic() || weights->get_output_shape(0) != weights_shape)
|
||||
return nullptr;
|
||||
if (auto constant = ov::as_type_ptr<ov::opset10::Constant>(weights)) {
|
||||
weights_to_concat.push_back(node_registry.make<ov::opset10::Constant>(*constant, new_shape));
|
||||
} else {
|
||||
weights_to_concat.push_back(node_registry.make<ov::opset10::Unsqueeze>(
|
||||
weights,
|
||||
ov::opset10::Constant::create(ov::element::i32, ov::Shape{}, {0})));
|
||||
}
|
||||
weights_to_concat.back().get_node()->set_friendly_name(weights->get_friendly_name());
|
||||
}
|
||||
|
||||
return node_registry.make<ov::opset10::Concat>(weights_to_concat, 0);
|
||||
}
|
||||
|
||||
ov::pass::ConvolutionToGroupConvolutionFusion::ConvolutionToGroupConvolutionFusion() {
|
||||
MATCHER_SCOPE(ConvolutionToGroupConvolutionFusion);
|
||||
|
||||
auto has_conv_inputs = [](const Output<Node>& node) -> bool {
|
||||
const auto concat = node.get_node();
|
||||
size_t num_inputs = concat->get_input_size();
|
||||
if (num_inputs == 0)
|
||||
return false;
|
||||
|
||||
const auto first_conv = as_type<opset10::Convolution>(concat->get_input_node_ptr(0));
|
||||
if (!first_conv)
|
||||
return false;
|
||||
|
||||
const auto split = first_conv->get_input_node_ptr(0);
|
||||
if (!is_type<opset10::Split>(split) && !is_type<opset10::VariadicSplit>(split))
|
||||
return false;
|
||||
|
||||
// go through Concat inputs and check
|
||||
// - if all of them are Convolutions
|
||||
// - if those Convolutions have the same Split input
|
||||
for (size_t i = 1; i < concat->get_input_size(); i++) {
|
||||
const auto conv = concat->get_input_node_ptr(i);
|
||||
if (conv->get_input_node_ptr(0) != split)
|
||||
return false;
|
||||
if (!compare_convolutions(first_conv, conv))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
auto concat_label = pattern::wrap_type<opset10::Concat>(has_conv_inputs);
|
||||
|
||||
matcher_pass_callback callback = [=](pattern::Matcher& m) {
|
||||
const auto& pattern_value_map = m.get_pattern_value_map();
|
||||
const auto& concat = pattern_value_map.at(concat_label).get_node_shared_ptr();
|
||||
|
||||
const auto first_conv = as_type_ptr<opset10::Convolution>(concat->get_input_node_shared_ptr(0));
|
||||
const auto split = first_conv->get_input_node_shared_ptr(0);
|
||||
const bool is_split = is_type<opset10::Split>(split);
|
||||
const bool is_variadic_split = is_type<opset10::VariadicSplit>(split);
|
||||
if (!is_split && !is_variadic_split)
|
||||
return false;
|
||||
|
||||
if (get_split_axis(split) != 1)
|
||||
return false;
|
||||
|
||||
if (is_variadic_split) {
|
||||
// split_lengths in VariadicSplit must have the same values
|
||||
if (auto split_lengths = as_type<opset10::Constant>(split->get_input_node_ptr(1))) {
|
||||
const auto split_lengths_values = split_lengths->cast_vector<int>();
|
||||
const auto first_length = split_lengths_values[0];
|
||||
if (!std::all_of(split_lengths_values.begin() + 1,
|
||||
split_lengths_values.end(),
|
||||
[first_length](int split_length) {
|
||||
return split_length == first_length;
|
||||
}))
|
||||
return false;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
NodeRegistry node_registry;
|
||||
const auto weights = create_new_weights(node_registry, concat);
|
||||
if (!weights)
|
||||
return false;
|
||||
|
||||
const auto conv = node_registry.make<opset10::GroupConvolution>(split->get_input_node_shared_ptr(0),
|
||||
weights,
|
||||
first_conv->get_strides(),
|
||||
first_conv->get_pads_begin(),
|
||||
first_conv->get_pads_end(),
|
||||
first_conv->get_dilations(),
|
||||
first_conv->get_auto_pad());
|
||||
conv->set_friendly_name(concat->get_friendly_name());
|
||||
register_new_node(conv);
|
||||
|
||||
const size_t concat_num_inputs = concat->get_input_size();
|
||||
NodeVector from;
|
||||
from.reserve(concat_num_inputs + 2);
|
||||
from.push_back(split);
|
||||
from.push_back(first_conv);
|
||||
for (size_t i = 1; i < concat_num_inputs; i++) {
|
||||
from.push_back(concat->get_input_node_shared_ptr(i));
|
||||
}
|
||||
from.push_back(concat);
|
||||
|
||||
copy_runtime_info(from, node_registry.get());
|
||||
replace_node(concat, conv);
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
auto m = std::make_shared<pattern::Matcher>(concat_label, matcher_name);
|
||||
this->register_matcher(m, callback);
|
||||
}
|
||||
@ -15,6 +15,7 @@
|
||||
#include <transformations/common_optimizations/conv_to_binary_conv.hpp>
|
||||
#include <transformations/common_optimizations/convert_nms_gather_path_to_unsigned.hpp>
|
||||
#include <transformations/common_optimizations/convert_quantize_dequantize.hpp>
|
||||
#include <transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp>
|
||||
#include <transformations/common_optimizations/depth_to_space_fusion.hpp>
|
||||
#include <transformations/common_optimizations/dilated_convolution_converter.hpp>
|
||||
#include <transformations/common_optimizations/disable_random_uniform_constant_folding.hpp>
|
||||
@ -189,6 +190,7 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr<ngraph::Fu
|
||||
ADD_MATCHER(common_fusions, RandomUniformFusion)
|
||||
ADD_MATCHER(common_fusions, ConvertTensorIteratorToSequence)
|
||||
ADD_MATCHER(common_fusions, SplitConcatPairToInterpolateFusion, m_use_shapes)
|
||||
ADD_MATCHER(common_fusions, ConvolutionToGroupConvolutionFusion)
|
||||
if (m_use_shapes) {
|
||||
ADD_MATCHER(common_fusions, NearestNeighborUpsamplingFusion)
|
||||
}
|
||||
|
||||
@ -0,0 +1,235 @@
|
||||
// Copyright (C) 2023 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <openvino/opsets/opset10.hpp>
|
||||
#include <openvino/pass/manager.hpp>
|
||||
#include <transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp>
|
||||
|
||||
#include "common_test_utils/ngraph_test_utils.hpp"
|
||||
|
||||
using namespace testing;
|
||||
using namespace ov;
|
||||
|
||||
TEST_F(TransformationTestsF, ConvToGroupConvFusionSplit) {
|
||||
Shape input_shape{2, 10, 14, 14};
|
||||
size_t num_splits = 5;
|
||||
int axis = 1;
|
||||
Shape weights_shape{3, input_shape[1] / num_splits, 1, 1};
|
||||
const auto spatial_dim_size = weights_shape.size() - 2;
|
||||
Strides strides(spatial_dim_size, 1);
|
||||
CoordinateDiff pads_begin(spatial_dim_size, 0);
|
||||
CoordinateDiff pads_end(spatial_dim_size, 0);
|
||||
Strides dilations(spatial_dim_size, 1);
|
||||
|
||||
{
|
||||
const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
|
||||
const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
|
||||
const auto split = std::make_shared<opset10::Split>(data, axis_node, num_splits);
|
||||
OutputVector concat_inputs;
|
||||
concat_inputs.reserve(num_splits);
|
||||
for (size_t i = 0; i < num_splits; i++) {
|
||||
const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1});
|
||||
concat_inputs.push_back(std::make_shared<opset10::Convolution>(split->output(i),
|
||||
weights,
|
||||
strides,
|
||||
pads_begin,
|
||||
pads_end,
|
||||
dilations));
|
||||
}
|
||||
const auto concat = std::make_shared<opset10::Concat>(concat_inputs, axis);
|
||||
function = std::make_shared<Model>(concat, ParameterVector{data});
|
||||
manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
|
||||
}
|
||||
|
||||
{
|
||||
const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
|
||||
OutputVector concat_inputs;
|
||||
concat_inputs.reserve(num_splits);
|
||||
Shape new_weights_shape = weights_shape;
|
||||
new_weights_shape.insert(new_weights_shape.begin(), 1);
|
||||
for (size_t i = 0; i < num_splits; i++) {
|
||||
const auto weights = opset10::Constant::create(element::f32, new_weights_shape, {i + 1});
|
||||
concat_inputs.push_back(weights);
|
||||
}
|
||||
const auto concat = std::make_shared<opset10::Concat>(concat_inputs, 0);
|
||||
const auto conv =
|
||||
std::make_shared<opset10::GroupConvolution>(data, concat, strides, pads_begin, pads_end, dilations);
|
||||
function_ref = std::make_shared<Model>(conv, ParameterVector{data});
|
||||
}
|
||||
|
||||
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
|
||||
comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
|
||||
}
|
||||
|
||||
TEST_F(TransformationTestsF, ConvToGroupConvFusionVariadicSplit) {
|
||||
Shape input_shape{2, 10, 14, 14};
|
||||
size_t num_splits = 5;
|
||||
int axis = 1;
|
||||
Shape weights_shape{3, input_shape[1] / num_splits, 1, 1};
|
||||
const auto spatial_dim_size = weights_shape.size() - 2;
|
||||
Strides strides(spatial_dim_size, 1);
|
||||
CoordinateDiff pads_begin(spatial_dim_size, 0);
|
||||
CoordinateDiff pads_end(spatial_dim_size, 0);
|
||||
Strides dilations(spatial_dim_size, 1);
|
||||
|
||||
{
|
||||
const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
|
||||
const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
|
||||
const auto split_lengths =
|
||||
opset10::Constant::create(element::i32, Shape{num_splits}, std::vector<int>(num_splits, 2));
|
||||
const auto split = std::make_shared<opset10::VariadicSplit>(data, axis_node, split_lengths);
|
||||
OutputVector concat_inputs;
|
||||
concat_inputs.reserve(num_splits);
|
||||
for (size_t i = 0; i < num_splits; i++) {
|
||||
const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1});
|
||||
concat_inputs.push_back(std::make_shared<opset10::Convolution>(split->output(i),
|
||||
weights,
|
||||
strides,
|
||||
pads_begin,
|
||||
pads_end,
|
||||
dilations));
|
||||
}
|
||||
const auto concat = std::make_shared<opset10::Concat>(concat_inputs, axis);
|
||||
function = std::make_shared<Model>(concat, ParameterVector{data});
|
||||
manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
|
||||
}
|
||||
|
||||
{
|
||||
const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
|
||||
OutputVector concat_inputs;
|
||||
concat_inputs.reserve(num_splits);
|
||||
Shape new_weights_shape = weights_shape;
|
||||
new_weights_shape.insert(new_weights_shape.begin(), 1);
|
||||
for (size_t i = 0; i < num_splits; i++) {
|
||||
const auto weights = opset10::Constant::create(element::f32, new_weights_shape, {i + 1});
|
||||
concat_inputs.push_back(weights);
|
||||
}
|
||||
const auto concat = std::make_shared<opset10::Concat>(concat_inputs, 0);
|
||||
const auto conv =
|
||||
std::make_shared<opset10::GroupConvolution>(data, concat, strides, pads_begin, pads_end, dilations);
|
||||
function_ref = std::make_shared<Model>(conv, ParameterVector{data});
|
||||
}
|
||||
|
||||
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
|
||||
comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
|
||||
}
|
||||
|
||||
TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionSplitInvalidAxis) {
|
||||
Shape input_shape{2, 10, 14, 14};
|
||||
int num_splits = 2;
|
||||
int axis = 2;
|
||||
Shape weights_shape{3, input_shape[1], 1, 1};
|
||||
const auto spatial_dim_size = weights_shape.size() - 2;
|
||||
Strides strides(spatial_dim_size, 1);
|
||||
CoordinateDiff pads_begin(spatial_dim_size, 0);
|
||||
CoordinateDiff pads_end(spatial_dim_size, 0);
|
||||
Strides dilations(spatial_dim_size, 1);
|
||||
|
||||
{
|
||||
const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
|
||||
const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
|
||||
const auto split = std::make_shared<opset10::Split>(data, axis_node, num_splits);
|
||||
OutputVector concat_inputs;
|
||||
concat_inputs.reserve(num_splits);
|
||||
for (int i = 0; i < num_splits; i++) {
|
||||
const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1});
|
||||
concat_inputs.push_back(std::make_shared<opset10::Convolution>(split->output(i),
|
||||
weights,
|
||||
strides,
|
||||
pads_begin,
|
||||
pads_end,
|
||||
dilations));
|
||||
}
|
||||
const auto concat = std::make_shared<opset10::Concat>(concat_inputs, axis);
|
||||
function = std::make_shared<Model>(concat, ParameterVector{data});
|
||||
manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
|
||||
}
|
||||
|
||||
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
|
||||
comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
|
||||
}
|
||||
|
||||
TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionSplitNotMatchingConvAttributes) {
|
||||
Shape input_shape{2, 10, 14, 14};
|
||||
size_t num_splits = 2;
|
||||
int axis = 1;
|
||||
const auto spatial_dim_size = 2;
|
||||
|
||||
{
|
||||
const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
|
||||
const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
|
||||
const auto split = std::make_shared<opset10::Split>(data, axis_node, num_splits);
|
||||
|
||||
const auto weights1 = opset10::Constant::create(element::f32, Shape{3, input_shape[1] / num_splits, 2, 2}, {1});
|
||||
Strides strides1(spatial_dim_size, 1);
|
||||
CoordinateDiff pads_begin1(spatial_dim_size, 1);
|
||||
CoordinateDiff pads_end1(spatial_dim_size, 1);
|
||||
Strides dilations1(spatial_dim_size, 1);
|
||||
const auto conv1 = std::make_shared<opset10::Convolution>(split->output(0),
|
||||
weights1,
|
||||
strides1,
|
||||
pads_begin1,
|
||||
pads_end1,
|
||||
dilations1);
|
||||
|
||||
const auto weights2 = opset10::Constant::create(element::f32, Shape{3, input_shape[1] / num_splits, 4, 4}, {1});
|
||||
Strides strides2(spatial_dim_size, 1);
|
||||
CoordinateDiff pads_begin2(spatial_dim_size, 2);
|
||||
CoordinateDiff pads_end2(spatial_dim_size, 2);
|
||||
Strides dilations2(spatial_dim_size, 1);
|
||||
const auto conv2 = std::make_shared<opset10::Convolution>(split->output(1),
|
||||
weights2,
|
||||
strides2,
|
||||
pads_begin2,
|
||||
pads_end2,
|
||||
dilations2);
|
||||
const auto concat = std::make_shared<opset10::Concat>(OutputVector{conv1, conv2}, axis);
|
||||
function = std::make_shared<Model>(concat, ParameterVector{data});
|
||||
manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
|
||||
}
|
||||
|
||||
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
|
||||
comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
|
||||
}
|
||||
|
||||
TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionVariadicSplitUnevenSplitLengths) {
|
||||
Shape input_shape{2, 10, 14, 14};
|
||||
int axis = 1;
|
||||
const auto spatial_dim_size = 2;
|
||||
Strides strides(spatial_dim_size, 1);
|
||||
CoordinateDiff pads_begin(spatial_dim_size, 0);
|
||||
CoordinateDiff pads_end(spatial_dim_size, 0);
|
||||
Strides dilations(spatial_dim_size, 1);
|
||||
|
||||
{
|
||||
const auto data = std::make_shared<opset10::Parameter>(element::f32, input_shape);
|
||||
const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis});
|
||||
const auto split_lengths = opset10::Constant::create(element::i32,
|
||||
Shape{2},
|
||||
std::vector<int>{3, static_cast<int>(input_shape[1]) - 3});
|
||||
const auto split = std::make_shared<opset10::VariadicSplit>(data, axis_node, split_lengths);
|
||||
const auto weights1 = opset10::Constant::create(element::f32, Shape{3, 3, 1, 1}, {1});
|
||||
const auto conv1 = std::make_shared<opset10::Convolution>(split->output(0),
|
||||
weights1,
|
||||
strides,
|
||||
pads_begin,
|
||||
pads_end,
|
||||
dilations);
|
||||
const auto weights2 = opset10::Constant::create(element::f32, Shape{3, 7, 1, 1}, {2});
|
||||
const auto conv2 = std::make_shared<opset10::Convolution>(split->output(1),
|
||||
weights2,
|
||||
strides,
|
||||
pads_begin,
|
||||
pads_end,
|
||||
dilations);
|
||||
const auto concat = std::make_shared<opset10::Concat>(OutputVector{conv1, conv2}, axis);
|
||||
function = std::make_shared<Model>(concat, ParameterVector{data});
|
||||
manager.register_pass<ov::pass::ConvolutionToGroupConvolutionFusion>();
|
||||
}
|
||||
|
||||
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
|
||||
comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user