From 586d97bcb0743cafce641cf6959dbba2adf08d31 Mon Sep 17 00:00:00 2001 From: Gleb Kazantaev Date: Sat, 11 Dec 2021 23:21:13 +0300 Subject: [PATCH] RIC Fusion Transformation (#8905) * Initial RIC Fusion version * Added automatic RIC inserter/erases * Updated RIC Attr; Added support for new operations; updated tests; simplified matcher registration * New operaitons; layout independency * Cleanup * Added FQ support * Cleanup; fix Conv+FQ * Added accuracy check; fixed GroupConvolution handler * Added Split->Concat and ShapeOf tests * Improved test coverege; cleanup * Function -> Model * Fix node usage * Added rank check and check for two consecutive init patterns --- .../transformations/ric_fusion_tests.cpp | 639 +++++++++++++++++ .../common_test_utils/CMakeLists.txt | 1 + .../common_test_utils/ngraph_test_utils.cpp | 242 +++++++ .../common_test_utils/ngraph_test_utils.hpp | 17 +- .../common_optimizations/ric_fusion.hpp | 30 + .../include/transformations/utils/utils.hpp | 33 +- .../moc_transformations.cpp | 3 + .../common_optimizations/ric_fusion.cpp | 661 ++++++++++++++++++ 8 files changed, 1623 insertions(+), 3 deletions(-) create mode 100644 inference-engine/tests/functional/inference_engine/transformations/ric_fusion_tests.cpp create mode 100644 inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp create mode 100644 src/common/transformations/include/transformations/common_optimizations/ric_fusion.hpp create mode 100644 src/common/transformations/src/transformations/common_optimizations/ric_fusion.cpp diff --git a/inference-engine/tests/functional/inference_engine/transformations/ric_fusion_tests.cpp b/inference-engine/tests/functional/inference_engine/transformations/ric_fusion_tests.cpp new file mode 100644 index 00000000000..cf56e34439e --- /dev/null +++ b/inference-engine/tests/functional/inference_engine/transformations/ric_fusion_tests.cpp @@ -0,0 +1,639 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common_test_utils/ngraph_test_utils.hpp" + +using namespace testing; +using namespace ngraph; +using namespace opset8; + +namespace { +std::shared_ptr create_conv(Output input, Output weights) { + return std::make_shared(input, weights, ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); +} + +std::shared_ptr create_weights(const Shape & weigts_shape) { + std::vector values(ov::shape_size(weigts_shape)); + float cur_value = 0.01; + for (auto & value : values) { + value = cur_value; + cur_value += 0.01; + } + return Constant::create(element::f32, weigts_shape, values); +} + +std::shared_ptr create_conv(Output input, const Shape & weigts_shape) { + return create_conv(input, create_weights(weigts_shape)); +} + +std::shared_ptr create_group_conv(Output input, Output weights) { + return std::make_shared(input, weights, ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); +} + +std::shared_ptr create_group_conv(Output input, const Shape & weigts_shape) { + return create_group_conv(input, create_weights(weigts_shape)); +} + +std::shared_ptr create_group_conv_with_gather(Output input, const Shape & weigts_shape, const std::vector & order) { + auto gather = std::make_shared(create_weights(weigts_shape), Constant::create(element::i64, Shape{order.size()}, order), + Constant::create(element::i64, Shape{1}, {0})); + return std::make_shared(input, gather, ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); +} + +std::shared_ptr create_conv_with_gather(Output input, const Shape & weigts_shape, const std::vector & order) { + auto gather = std::make_shared(create_weights(weigts_shape), Constant::create(element::i64, Shape{order.size()}, order), + Constant::create(element::i64, Shape{1}, {1})); + return std::make_shared(input, gather, ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); +} + +std::shared_ptr create_param(const PartialShape & shape) { + return std::make_shared(element::f32, shape); +} + +std::shared_ptr create_gather(Output input, std::vector order, int64_t axis) { + auto order_const = Constant::create(element::i64, Shape{order.size()}, order); + auto axis_const = Constant::create(element::i64, Shape{1}, {axis}); + return std::make_shared(input, order_const, axis_const); +} + +std::shared_ptr create_fq(Output input) { + return std::make_shared(input, Constant::create(element::f32, Shape{1}, {1}), + Constant::create(element::f32, Shape{1}, {2}), + Constant::create(element::f32, Shape{1}, {3}), + Constant::create(element::f32, Shape{1}, {4}), 255); +} + +void apply_reverse_input_channels(std::shared_ptr f, std::vector> data) { + using namespace ov::preprocess; + PrePostProcessor p(f); + for (auto item : data) { + p.input(item.first).tensor().set_layout(ov::Layout(item.second)); + p.input(item.first).preprocess().reverse_channels(); + } + p.build(); +} +} // namespace + +TEST_F(TransformationTestsF, RICFusionSimple) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto relu = std::make_shared(input); + auto conv = create_conv(relu, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({ 1, 3, 64, 64 }); + auto relu = std::make_shared(input); + auto conv = create_conv_with_gather(relu, {6, 3, 3, 3}, {2, 1, 0}); + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionHard) { + { + auto input = create_param({ -1, 3, -1, -1 }); + auto relu = std::make_shared(input); + + auto input2 = create_param({ -1, 3, -1, -1 }); + auto split = std::make_shared(input2, Constant::create(element::i64, {}, {1}), 3); + auto concat = std::make_shared(OutputVector{split->output(2), split->output(1), split->output(0)}, 1); + auto eltwise = std::make_shared(relu, concat); + auto prelu = std::make_shared(eltwise, Constant::create(element::f32, Shape{}, {6.0f})); + + auto gconv = create_group_conv(prelu, {3, 4, 1, 3, 3}); + + auto pow = std::make_shared(gconv, Constant::create(element::f32, Shape{}, {-1.0f})); + auto convert1 = std::make_shared(pow, element::f16); + auto convert2 = std::make_shared(convert1, element::f32); + + auto gconv2 = create_group_conv(convert2, {12, 1, 1, 3, 3}); + + auto conv = create_conv(gconv2, {6, 12, 3, 3}); + auto conv2 = create_conv(concat, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv, conv2 }, ParameterVector{ input, input2 }); + + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + manager.register_pass(); + } + { + auto input = create_param({ -1, 3, -1, -1 }); + auto relu = std::make_shared(input); + + auto input2 = create_param({ -1, 3, -1, -1 }); + auto eltwise = std::make_shared(relu, input2); + auto prelu = std::make_shared(eltwise, Constant::create(element::f32, Shape{}, {6.0f})); + // 0 1 2 2 1 0 + // [0, 1, 2, 3]-[4, 5, 6, 7]-[8, 9, 10, 11] -> [8, 9, 10, 11]-[4, 5, 6, 7]-[0, 1, 2, 3] + auto gconv = create_group_conv_with_gather(prelu, {3, 4, 1, 3, 3}, {2, 1, 0}); + + auto pow = std::make_shared(gconv, Constant::create(element::f32, Shape{}, {-1.0f})); + auto convert1 = std::make_shared(pow, element::f16); + auto convert2 = std::make_shared(convert1, element::f32); + + auto gconv2 = create_group_conv_with_gather(convert2, {12, 1, 1, 3, 3}, {8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3}); + + auto conv = create_conv_with_gather(gconv2, {6, 12, 3, 3}, {8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3}); + auto conv2 = create_conv_with_gather(input2, {6, 3, 3, 3}, {2, 1, 0}); + + function_ref = std::make_shared(NodeVector{ conv, conv2 }, ParameterVector{ input, input2 }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionEltwise1) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(input, Constant::create(element::f32, Shape{3, 1, 1}, {0.1, 0.2, 0.3})); + auto conv = create_conv(add, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gather = create_gather(Constant::create(element::f32, Shape{3, 1, 1}, {0.1, 0.2, 0.3}), {2, 1, 0}, 0); + auto add = std::make_shared(input, gather); + auto conv = create_conv_with_gather(add, {6, 3, 3, 3}, {2, 1, 0}); + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionEltwise2) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(input, create_weights({1, 1, 1})); + auto conv = create_conv(add, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(input, create_weights({1, 1, 1})); + auto conv = create_conv_with_gather(add, {6, 3, 3, 3}, {2, 1, 0}); + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionEltwise3) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(input, Constant::create(element::f32, Shape{1}, {0.2})); + auto conv = create_conv(add, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(input, Constant::create(element::f32, Shape{1}, {0.2})); + auto conv = create_conv_with_gather(add, {6, 3, 3, 3}, {2, 1, 0}); + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionEltwise4) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(create_weights({3, 1, 1}), input); + auto conv = create_conv(add, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gather = create_gather(create_weights({3, 1, 1}), {2, 1, 0}, 0); + auto add = std::make_shared(gather, input); + auto conv = create_conv_with_gather(add, {6, 3, 3, 3}, {2, 1, 0}); + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionEltwise5) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(create_weights({1, 3, 1, 1}), input); + auto conv = create_conv(add, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gather = create_gather(create_weights({1, 3, 1, 1}), {2, 1, 0}, 1); + auto add = std::make_shared(gather, input); + auto conv = create_conv_with_gather(add, {6, 3, 3, 3}, {2, 1, 0}); + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionEltwiseNegative) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto input2 = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(input, input2); + auto conv = create_conv(add, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input, input2 }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionEltwiseNegative2) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto input2 = create_param({ 1, 1, 64, 64 }); + auto add = std::make_shared(input, input2); + auto conv = create_conv(add, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input, input2 }); + apply_reverse_input_channels(function, {{0, "NCHW"}, {1, "NCHW"}}); + + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionEltwiseNegative3) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto add = std::make_shared(input, Constant::create(element::f32, {1, 1, 1, 1, 1}, {1.4})); + auto shapeof = std::make_shared(add); + + function = std::make_shared(NodeVector{ shapeof }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionGroupConv) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gconv = create_group_conv(input, {3, 2, 1, 3, 3}); + auto relu = std::make_shared(gconv); + auto conv = create_conv(relu, {3, 6, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gconv = create_group_conv_with_gather(input, {3, 2, 1, 3, 3}, {2, 1, 0}); + // 0 1 2 2 1 0 + // [0, 1]-[2, 3]-[4, 5] -> [4, 5]-[2, 3]-[0, 1] + auto relu = std::make_shared(gconv); + auto conv = create_conv_with_gather(relu, {3, 6, 3, 3}, {4, 5, 2, 3, 0, 1}); + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionGroupConvNegative) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gconv = create_group_conv(input, {1, 3, 3, 3, 3}); + auto relu = std::make_shared(gconv); + auto conv = create_conv(relu, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionTranspose) { + { + auto input = create_param({ 1, 64, 64, 3 }); + auto add = std::make_shared(input, create_weights({3})); + auto transpose = std::make_shared(add, Constant::create(element::i64, Shape{4}, {0, 3, 1, 2})); + auto conv = create_conv(transpose, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NHWC"}}); + + manager.register_pass(); + } + + { + auto input = create_param({ 1, 64, 64, 3 }); + auto gather = create_gather(create_weights({3}), {2, 1, 0}, 0); + auto add = std::make_shared(input, gather); + auto transpose = std::make_shared(add, Constant::create(element::i64, Shape{4}, {0, 3, 1, 2})); + auto conv = create_conv_with_gather(transpose, {6, 3, 3, 3}, {2, 1, 0}); + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionFQOnTheWay) { + { + auto input = create_param({1, 3, 64, 64}); + auto fq = create_fq(input); + auto conv = create_conv(fq, create_fq(create_weights({6, 3, 3, 3}))); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({1, 3, 64, 64}); + auto fq = create_fq(input); + auto conv = create_conv(fq, create_fq(create_gather(create_weights({6, 3, 3, 3}), {2, 1, 0}, 1))); + + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionFQOnTheWay2) { + { + auto input = create_param({1, 3, 64, 64}); + auto fq = create_fq(input); + auto weights_const = create_weights({6, 3, 3, 3}); + auto fq_weights = std::make_shared(weights_const, + create_weights({1, 3, 1, 1}), + create_weights({1, 1, 1}), + create_weights({1}), + create_weights({3, 1, 1}), 255); + auto conv = create_conv(fq, fq_weights); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({1, 3, 64, 64}); + auto fq = create_fq(input); + auto weights_const = create_weights({6, 3, 3, 3}); + auto fq_weights = std::make_shared(create_gather(weights_const, {2, 1, 0}, 1), + create_gather(create_weights({1, 3, 1, 1}), {2, 1, 0}, 1), + create_weights({1, 1, 1}), + create_weights({1}), + create_gather(create_weights({3, 1, 1}), {2, 1, 0}, 0), + 255); + auto conv = create_conv(fq, fq_weights); + + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionFQOnTheWay3) { + { + auto input = create_param({1, 3, 64, 64}); + auto fq = create_fq(input); + auto weights_const = create_weights({3, 1, 1, 3, 3}); + auto fq_weights = std::make_shared(weights_const, + create_weights({3, 1, 1, 1, 1}), + create_weights({1, 1, 1}), + create_weights({1}), + create_weights({1}), 255); + auto gconv = create_group_conv(fq, fq_weights); + auto conv = create_conv(gconv, {6, 3, 1, 1}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({1, 3, 64, 64}); + auto fq = create_fq(input); + auto weights_const = create_weights({3, 1, 1, 3, 3}); + auto fq_weights = std::make_shared(create_gather(weights_const, {2, 1, 0}, 0), + create_gather(create_weights({3, 1, 1, 1, 1}), {2, 1, 0}, 0), + create_weights({1, 1, 1}), + create_weights({1}), + create_weights({1}), 255); + auto gconv = create_group_conv(fq, fq_weights); + auto conv = create_conv_with_gather(gconv, {6, 3, 1, 1}, {2, 1, 0}); + + function_ref = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); + enable_accuracy_check(); +} + +TEST_F(TransformationTestsF, RICFusionShapeOf) { + { + auto input = create_param({1, 3, 64, 64}); + auto relu = std::make_shared(input); + auto shape_of = std::make_shared(relu); + + function = std::make_shared(NodeVector{ shape_of }, ParameterVector{ input }); + apply_reverse_input_channels(function, {{0, "NCHW"}}); + + manager.register_pass(); + } + + { + auto input = create_param({1, 3, 64, 64}); + auto relu = std::make_shared(input); + auto shape_of = std::make_shared(relu); + + function_ref = std::make_shared(NodeVector{ shape_of }, ParameterVector{ input }); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + disable_rt_info_check(); +} + +TEST_F(TransformationTestsF, RICFusionGatherDetectionNegative) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gather = create_gather(input, {2, 1, 1}, 1); + auto relu = std::make_shared(gather); + auto conv = create_conv(relu, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionGatherDetectionNegative2) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto input2 = std::make_shared(element::i64, Shape{3}); + auto gather = std::make_shared(input, input2, Constant::create(element::i64, {}, {1})); + auto relu = std::make_shared(gather); + auto conv = create_conv(relu, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input, input2 }); + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionGatherDetectionNegative3) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gather = create_gather(input, {1}, 1); + auto relu = std::make_shared(gather); + auto conv = create_conv(relu, {6, 1, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionGatherDetectionNegative4) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto gather = create_gather(input, {2, 0}, 1); + auto relu = std::make_shared(gather); + auto conv = create_conv(relu, {6, 2, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionSplitConcatDetectionNegative) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto split = std::make_shared(input, Constant::create(element::i64, {}, {1}), 3); + auto concat = std::make_shared(OutputVector{split->output(2), split->output(1), split->output(1)}, 1); + auto relu = std::make_shared(concat); + auto conv = create_conv(relu, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionSplitConcatDetectionNegative2) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto split = std::make_shared(input, Constant::create(element::i64, {}, {1}), 3); + auto split2 = std::make_shared(input, Constant::create(element::i64, {}, {1}), 3); + auto concat = std::make_shared(OutputVector{split->output(2), split->output(1), split2->output(0)}, 1); + auto relu = std::make_shared(concat); + auto conv = create_conv(relu, {6, 3, 3, 3}); + + function = std::make_shared(NodeVector{ conv }, ParameterVector{ input }); + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, RICFusionSplitConcatDetectionNegative3) { + { + auto input = create_param({ 1, 3, 64, 64 }); + auto split = std::make_shared(input, Constant::create(element::i64, {}, {1}), 3); + auto concat = std::make_shared(OutputVector{split->output(2), split->output(1), split->output(0)}, 1); + auto relu = std::make_shared(concat); + auto conv = create_conv(relu, {6, 3, 3, 3}); + + function = std::make_shared(OutputVector{ conv, split->output(0) }, ParameterVector{ input }); + manager.register_pass(); + } +} diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/CMakeLists.txt b/inference-engine/tests/ie_test_utils/common_test_utils/CMakeLists.txt index 3de4a2ac827..009a222e19a 100644 --- a/inference-engine/tests/ie_test_utils/common_test_utils/CMakeLists.txt +++ b/inference-engine/tests/ie_test_utils/common_test_utils/CMakeLists.txt @@ -20,6 +20,7 @@ function(add_common_utils ADD_TARGET_NAME) gtest_main inference_engine_transformations ngraph_test_util + ngraphFunctions PRIVATE openvino::util ) diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp b/inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp new file mode 100644 index 00000000000..88c8e42cdeb --- /dev/null +++ b/inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp @@ -0,0 +1,242 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "ngraph_test_utils.hpp" +#include + +namespace { + +template +static void Compare(const T_NGRAPH *expected, const T_IE *actual, std::size_t size, float threshold, float abs_threshold = -1.f) { + for (std::size_t i = 0; i < size; ++i) { + const T_NGRAPH &ref = expected[i]; + const auto &res = actual[i]; + const auto absoluteDifference = CommonTestUtils::ie_abs(res - ref); + if (abs_threshold > 0.f && absoluteDifference > abs_threshold) { + IE_THROW() << "Absolute comparison of values expected: " << std::to_string(ref) << " and actual: " << std::to_string(res) + << " at index " << i << " with absolute threshold " << abs_threshold + << " failed"; + } + if (absoluteDifference <= threshold) { + continue; + } + double max; + if (sizeof(T_IE) < sizeof(T_NGRAPH)) { + max = std::max(CommonTestUtils::ie_abs(T_NGRAPH(res)), CommonTestUtils::ie_abs(ref)); + } else { + max = std::max(CommonTestUtils::ie_abs(res), CommonTestUtils::ie_abs(T_IE(ref))); + } + double diff = static_cast(absoluteDifference) / max; + if (max == 0 || (diff > static_cast(threshold)) || + (std::isnan(static_cast(res)) ^ std::isnan(static_cast(ref)))) { + IE_THROW() << "Relative comparison of values expected: " << std::to_string(ref) << " and actual: " << std::to_string(res) + << " at index " << i << " with threshold " << threshold + << " failed"; + } + } +} + +template +void callCompare(const std::pair> &expected, + const T_IE* actualBuffer, size_t size, float threshold, float abs_threshold) { + auto expectedBuffer = expected.second.data(); + switch (expected.first) { + case ngraph::element::Type_t::i64: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::i32: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::i16: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::i8: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::u64: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::u32: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::u16: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::boolean: + case ngraph::element::Type_t::u8: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::f64: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::f32: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::f16: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; + case ngraph::element::Type_t::bf16: + Compare(reinterpret_cast(expectedBuffer), + actualBuffer, size, threshold, abs_threshold); + break; +// case ngraph::element::Type_t::i4: { +// auto expectedOut = ngraph::helpers::convertOutputPrecision( +// expected.second, +// expected.first, +// ngraph::element::Type_t::i8, +// size); +// Compare(reinterpret_cast(expectedOut.data()), +// actualBuffer, size, threshold, abs_threshold); +// break; +// } +// case ngraph::element::Type_t::u4: { +// auto expectedOut = ngraph::helpers::convertOutputPrecision( +// expected.second, +// expected.first, +// ngraph::element::Type_t::u8, +// size); +// Compare(reinterpret_cast(expectedOut.data()), +// actualBuffer, size, threshold, abs_threshold); +// break; +// } + case ngraph::element::Type_t::dynamic: + case ngraph::element::Type_t::undefined: + Compare(reinterpret_cast(expectedBuffer), actualBuffer, size, threshold, abs_threshold); + break; + default: FAIL() << "Comparator for " << expected.first << " precision isn't supported"; + } + return; +} + + +void Compare(const std::pair> &expected, + const InferenceEngine::Blob::Ptr &actual, + float threshold, + float abs_threshold) { + const auto &precision = actual->getTensorDesc().getPrecision(); + auto k = static_cast(expected.first.size()) / precision.size(); + // W/A for int4, uint4 + if (expected.first == ngraph::element::Type_t::u4 || expected.first == ngraph::element::Type_t::i4) { + k /= 2; + } else if (expected.first == ngraph::element::Type_t::undefined || expected.first == ngraph::element::Type_t::dynamic) { + k = 1; + } + ASSERT_EQ(expected.second.size(), actual->byteSize() * k); + + auto memory = InferenceEngine::as(actual); + IE_ASSERT(memory); + const auto lockedMemory = memory->wmap(); + const auto actualBuffer = lockedMemory.as(); + + const auto &size = actual->size(); + switch (precision) { + case InferenceEngine::Precision::FP32: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::I32: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::I64: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::I8: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::U16: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::I16: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::BOOL: + case InferenceEngine::Precision::U8: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::U64: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::BF16: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + case InferenceEngine::Precision::FP16: + callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); + break; + default: + FAIL() << "Comparator for " << precision << " precision isn't supported"; + } +} + +void Compare(const std::vector>> &expectedOutputs, + const std::vector &actualOutputs, + float threshold, float abs_threshold) { + for (std::size_t outputIndex = 0; outputIndex < expectedOutputs.size(); ++outputIndex) { + const auto &expected = expectedOutputs[outputIndex]; + const auto &actual = actualOutputs[outputIndex]; + Compare(expected, actual, threshold, abs_threshold); + } +} +} // namespace + +void TransformationTestsF::accuracy_check(std::shared_ptr ref_function, + std::shared_ptr cur_function) { + try { + if (ref_function->is_dynamic() || cur_function->is_dynamic()) { + return; + } + std::vector> input_data; + ngraph::element::TypeVector types; + for (auto param : ref_function->get_parameters()) { + types.push_back(param->get_element_type()); + + InferenceEngine::TensorDesc td(InferenceEngine::Precision::FP32, param->get_shape(), InferenceEngine::Layout::ANY); + const auto &input = FuncTestUtils::createAndFillBlob(td); + const auto &input_size = input->byteSize(); + + std::vector data; + data.resize(input_size); + + auto memory = InferenceEngine::as(input); + IE_ASSERT(memory); + + const auto lockedMemory = memory->wmap(); + const auto buffer = lockedMemory.as(); + std::copy(buffer, buffer + input_size, data.data()); + + input_data.push_back(std::move(data)); + } + + auto ref_outputs = ngraph::helpers::interpreterFunction(ref_function, input_data, types); + auto outputs = ngraph::helpers::interpreterFunction(cur_function, input_data, types); + + IE_ASSERT(ref_outputs.size() == outputs.size()); + + for (size_t i = 0; i < ref_outputs.size(); ++i) { + IE_ASSERT(ref_outputs[i].second.size() == outputs[i].second.size()); + auto * ref = reinterpret_cast(ref_outputs[i].second.data()); + auto * out = reinterpret_cast(outputs[i].second.data()); + IE_ASSERT(ref_outputs[i].second.size() / 8); + size_t size = ref_outputs[i].second.size() / sizeof(float); + Compare(ref, out, size, 1e-5); + } + } + catch (const std::runtime_error &re) { + GTEST_FATAL_FAILURE_(re.what()); + } catch (const std::exception &ex) { + GTEST_FATAL_FAILURE_(ex.what()); + } catch (...) { + GTEST_FATAL_FAILURE_("Unknown failure occurred."); + } +} diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp b/inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp index a3d33ccda3c..3437508c180 100644 --- a/inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp +++ b/inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "ie_common.h" @@ -44,8 +45,9 @@ public: } void TearDown() override { + auto cloned_function = ngraph::clone_function(*function); if (!function_ref) { - function_ref = ngraph::clone_function(*function); + function_ref = cloned_function; } manager.register_pass(m_unh, m_soft_names_comparison); @@ -56,6 +58,10 @@ public: auto res = comparator.compare(function, function_ref); ASSERT_TRUE(res.valid) << res.message; + + if (m_enable_accuracy_check) { + accuracy_check(cloned_function, function); + } } // TODO: this is temporary solution to disable rt info checks that must be applied by default @@ -68,7 +74,13 @@ public: m_soft_names_comparison = true; } - std::shared_ptr function, function_ref; + void enable_accuracy_check() { + m_enable_accuracy_check = true; + } + + void accuracy_check(std::shared_ptr ref_function, std::shared_ptr cur_function); + + std::shared_ptr function, function_ref; ngraph::pass::Manager manager; FunctionsComparator comparator; @@ -76,4 +88,5 @@ private: std::shared_ptr m_unh; bool m_disable_rt_info_check{false}; bool m_soft_names_comparison{true}; + bool m_enable_accuracy_check{false}; }; diff --git a/src/common/transformations/include/transformations/common_optimizations/ric_fusion.hpp b/src/common/transformations/include/transformations/common_optimizations/ric_fusion.hpp new file mode 100644 index 00000000000..475bf4bdfae --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/ric_fusion.hpp @@ -0,0 +1,30 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include +#include +#include + +namespace ngraph { +namespace pass { + +class TRANSFORMATIONS_API ReverseInputChannelsFusion; + +} // namespace pass +} // namespace ngraph + +/** + * @ingroup ie_transformation_common_api + * @brief ReverseInputChannelsFusion + */ + +class ngraph::pass::ReverseInputChannelsFusion: public ov::pass::ModelPass { +public: + NGRAPH_RTTI_DECLARATION; + bool run_on_model(const std::shared_ptr &) override; +}; diff --git a/src/common/transformations/include/transformations/utils/utils.hpp b/src/common/transformations/include/transformations/utils/utils.hpp index 61d8f2ef7e7..59180a83230 100644 --- a/src/common/transformations/include/transformations/utils/utils.hpp +++ b/src/common/transformations/include/transformations/utils/utils.hpp @@ -92,9 +92,14 @@ inline std::string get_ie_output_name(const ngraph::Output& output } template -bool has_constant_value(const std::shared_ptr& constant, +bool has_constant_value(const std::shared_ptr& node, const T value, T epsilon = std::numeric_limits::epsilon()) { + if (!node) { + return false; + } + + auto constant = std::dynamic_pointer_cast(node); if (!constant) { return false; } @@ -123,6 +128,32 @@ bool has_constant_value(const std::shared_ptr& constan return true; } +template +bool has_constant_value(const std::shared_ptr& node, + const std::vector values, + T epsilon = std::numeric_limits::epsilon()) { + if (!node) { + return false; + } + + auto constant = std::dynamic_pointer_cast(node); + if (!constant) { + return false; + } + + const auto const_values = constant->cast_vector(); + + if (constant->get_element_type() == ngraph::element::f16 || + constant->get_element_type() == ngraph::element::f32 || + constant->get_element_type() == ngraph::element::f64 || + constant->get_element_type() == ngraph::element::bf16) { + return std::equal(const_values.cbegin(), const_values.cend(), values.cbegin(), + [&] (T lhs, T rhs) { return std::fabs(lhs - rhs) < epsilon; }); + } + + return const_values == values; +} + TRANSFORMATIONS_API bool get_single_value(const std::shared_ptr & const_node, float & value); TRANSFORMATIONS_API std::shared_ptr normalize_constant(const std::shared_ptr & constant, diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index b34146a87ab..e2bb9998125 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -56,6 +56,7 @@ #include #include #include +#include NGRAPH_RTTI_DEFINITION(ngraph::pass::MOCTransformations, "MOCTransformations", 0); @@ -174,6 +175,8 @@ bool ngraph::pass::MOCTransformations::run_on_model(const std::shared_ptradd_matcher(); conv_fusions->set_name("ngraph::pass::ConvFusions"); + manager.register_pass(); + manager.register_pass(); manager.run_passes(f); diff --git a/src/common/transformations/src/transformations/common_optimizations/ric_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/ric_fusion.cpp new file mode 100644 index 00000000000..8f6272617ea --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/ric_fusion.cpp @@ -0,0 +1,661 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/ric_fusion.hpp" +#include "transformations/utils/utils.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "itt.hpp" + + +NGRAPH_RTTI_DEFINITION(ngraph::pass::ReverseInputChannelsFusion, "ReverseInputChannelsFusion", 0); + +namespace ngraph { +namespace pass { +namespace ric_attr { + +// Attribute describes RIC type which we propagate. +// Also, it contains callback which can expand this attribute to the real RIC sub-graph. +// In addition, attribute has some functionality and properties for propagation. +class Attribute { +public: + using callback_t = std::function, const Attribute &)>; + + Attribute(std::vector order, int64_t axis, bool is_final = false, bool is_initial = false) + : m_order(std::move(order)), m_axis(axis), m_is_final(is_final), m_is_initial(is_initial) { + m_can_be_fused.emplace_back(std::make_shared(true)); + } + + // Method which is used to create a copy of attribute for further propagation. + // TODO: can be removed and replaced with regular copy but we need to get rid of + // is_initial flag and use some other way to detect original RIC output. + Attribute propagate() const { + Attribute attr(m_order, m_axis); + attr.m_can_be_fused = m_can_be_fused; + return attr; + } + + void set_is_final(bool is_final) { m_is_final = is_final; } + + void set_can_be_fused(bool can_be_fused) { + std::for_each(m_can_be_fused.cbegin(), m_can_be_fused.cend(), + [can_be_fused](const std::shared_ptr & state) { + *state = can_be_fused; + }); + } + + void set_callback(callback_t callback) { + m_callback = std::move(callback); + } + + // Apply callback to materialize RIC inside graph + void operator() (Input input) const { + m_callback(input, *this); + } + + bool can_be_fused() const { + return std::all_of(m_can_be_fused.cbegin(), m_can_be_fused.cend(), + [](const std::shared_ptr & state) { + return *state; + }); + } + + // For cases when we propagate through operation with multiple inputs like Eltwise + // we have to merge RIC attrs from all inputs. To check that given attr be merged with + // current we check the order and axis which must be the same. + bool can_be_merged_with(const Attribute & other) { + return m_order == other.m_order && m_axis == other.m_axis; + } + + // When merging two and more attrs for further propagation we have to keep can_be_fused references + // for cases when fusion is not possible, so we can update all related attrs. + void merge_with(const Attribute & other) { + m_can_be_fused.insert(m_can_be_fused.end(), + other.m_can_be_fused.begin(), + other.m_can_be_fused.end()); + } + + const std::vector & get_order() const { return m_order; } + + void set_order(const std::vector & order) { m_order = order; } + + int64_t get_axis() const { return m_axis; } + + void set_axis(int64_t axis) { m_axis = axis; } + + bool is_final() const { return m_is_final; } + + bool is_initial() const { return m_is_initial; } + +private: + std::vector m_order; + int64_t m_axis; + + // Specifies whether RIC can be fused or not. vector is needed to keep references to other + // attributes that were participated during merge. + std::vector> m_can_be_fused; + + // true - means that current RIC attribute is final and can be materialized + // false - means that current RIC attribute is temporary and need only for propagation + bool m_is_final; + + // true - means that current RIC attribute is an initial attribute and belongs to real RIC output + // false - means that current RIC attribute is temporary and need only for propagation + bool m_is_initial; + + // Callback specifies the action for RIC materialization for given input port. + // In most cases it should insert Gather operation for the input. + std::function, const Attribute &)> m_callback = + [](Input, const Attribute &) {}; +}; + +namespace { + +template +using is_port = typename std::enable_if>::value>::type; + +template > +void set(T port, const Attribute & ric_attr) { + auto & attrs = port.get_rt_info(); + attrs["reverse_input_channel_index"] = ric_attr; +} + +// Available only for output ports +void init(Output output, std::vector order, int64_t axis) { + set(output, Attribute(std::move(order), axis, false, true)); +} + +template > +bool has(const T & port) { + const auto & attrs = port.get_rt_info(); + return attrs.count("reverse_input_channel_index"); +} + +template > +Attribute get(const T & port) { + const auto & attrs = port.get_rt_info(); + auto res = attrs.find("reverse_input_channel_index"); + if (res != attrs.end()) { + return res->second.template as(); + } + throw ngraph_error("reverse_input_channel_index is missing in given port"); +} + +template > +void erase(T port) { + auto & rt_info = port.get_rt_info(); + rt_info.erase("reverse_input_channel_index"); +} +}// namespace +}// namespace ric_attr + +namespace init { +class SplitConcat : public ngraph::pass::MatcherPass { +public: + SplitConcat() { + MATCHER_SCOPE(SplitConcat); + auto split_p = pattern::wrap_type(); + auto pattern_root = pattern::wrap_type({split_p, split_p, split_p}); + + auto callback = [=](pattern::Matcher& m) { + const auto & pattern_map = m.get_pattern_value_map(); + auto concat = ov::as_type_ptr(pattern_map.at(pattern_root).get_node_shared_ptr()); + auto split = ov::as_type_ptr(pattern_map.at(split_p).get_node_shared_ptr()); + if (!concat || !split) return false; + + // Avoid cases with two consecutive Split->Concat + if (ric_attr::has(split->input_value(0))) { + return false; + } + + std::vector order; + order.reserve(split->get_num_splits()); + + for (const auto & input : concat->inputs()) { + auto split_output = input.get_source_output(); + if (split_output.get_node() != split.get()) return false; + + // Check that Concat is the only Split consumer and order of Split outputs + // satisfies expected order for reverse input channel case. + for (const auto & target_input : split_output.get_target_inputs()) { + if (target_input.get_node() != concat.get()) { + return false; + } + order.emplace_back(split_output.get_index()); + } + } + + // Check that all order values are unique, otherwise it is not RIC + std::set unique_values(order.cbegin(), order.cend()); + if (unique_values.size() != order.size()) { + return false; + } + + // Mark-up RIC output + ric_attr::init(concat, order, concat->get_axis()); + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class Gather : public ngraph::pass::MatcherPass { +public: + Gather() { + MATCHER_SCOPE(Gather); + auto input_p = pattern::any_input(pattern::has_static_rank()); + auto indices_p = pattern::any_input(); + auto axis_p = pattern::wrap_type(); + auto pattern_root = pattern::wrap_type({input_p, indices_p, axis_p}); + + auto callback = [=](pattern::Matcher& m) { + const auto & pattern_map = m.get_pattern_value_map(); + const auto & output = pattern_map.at(pattern_root); + + auto order = ov::get_constant_from_source(pattern_map.at(indices_p)); + auto axis = ov::get_constant_from_source(pattern_map.at(axis_p)); + if (!order || !axis) { + return false; + } + + // Avoid cases with two consecutive Gathers + if (ric_attr::has(pattern_map.at(input_p))) { + return false; + } + + const auto axis_value = axis->cast_vector().at(0); + + // This constraint helps to avoid detection of other Gathers that do not perform RIC + const auto & data_shape = m.get_match_root()->input(0).get_partial_shape(); + if (shape_size(order->get_shape()) == 1 || + axis_value < 0 || + axis_value >= data_shape.rank().get_length() || + data_shape[axis_value].is_dynamic() || + shape_size(order->get_shape()) != static_cast(data_shape[axis_value].get_length())) { + return false; + } + + // Check that all order values are unique, otherwise it is not RIC + const auto & order_values = order->cast_vector(); + std::set unique_values(order_values.cbegin(), order_values.cend()); + if (unique_values.size() != order_values.size()) { + return false; + } + ric_attr::init(output, order_values, axis_value); + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; +}// namespace init + +namespace prop { +namespace { +std::shared_ptr create_const(const std::vector & values) { + return opset8::Constant::create(ov::element::i64, ov::Shape{values.size()}, values); +} +}// namespace + +class Binary : public ngraph::pass::MatcherPass { +public: + Binary() { + MATCHER_SCOPE(Binary); + auto pattern_root = pattern::wrap_type(); + + auto callback = [=](pattern::Matcher& m) { + const auto & root = m.get_match_root(); + const auto & inputs = root->inputs(); + + std::map attrs; + for (const auto & input : inputs) { + auto output = input.get_source_output(); + if (ric_attr::has(output)) { + attrs.insert({input.get_index(), ric_attr::get(output).propagate()}); + } else if (!ov::is_type(output.get_node())) { + // If number of non-constant inputs and without RIC attr is greater than 0 we have to skip + // propagation because it is not efficient to have a lot of RIC copies on data path. + return false; + } + } + + if (attrs.empty()) return false; + + // Check that all RIC attrs can be merged and then merge them + auto ric = attrs.begin()->second; + auto rank = root->get_input_partial_shape(attrs.begin()->first).rank(); + if (rank.is_dynamic()) return false; + auto data_rank = rank.get_length(); + + for (const auto & item : attrs) { + const auto & input_rank = root->get_input_partial_shape(item.first).rank(); + if (input_rank.is_static() && + input_rank.get_length() == data_rank && + ric.can_be_merged_with(item.second) ) { + ric.merge_with(item.second); + } else { + return false; + } + } + + for (const auto & input : inputs) { + // Skip input that have RIC attribute + if (attrs.count(input.get_index())) continue; + + auto const_output = input.get_source_output(); + const auto & shape = const_output.get_shape(); + const int64_t & shape_rank = static_cast(shape.size()); + if (shape_rank > data_rank) { + // TODO: handle case when constant input broadcast another one + return false; + } + + if (data_rank - shape_rank > ric.get_axis()) { + // we don't have to insert RIC for constant, so we keep propagating + ric_attr::set(m.get_match_value(), ric); + continue; + } + + const int64_t & new_axis = ric.get_axis() - (data_rank - shape_rank); + if (shape[new_axis] == 1) { + // we don't have to insert RIC for constant, so we keep propagating + ric_attr::set(m.get_match_value(), ric); + continue; + } + + // finally, insert RIC + auto ric_const = ric; + ric_const.set_axis(new_axis); + ric_const.set_is_final(true); + ric_const.set_callback([](Input input, const ric_attr::Attribute & attr) { + auto output = input.get_source_output(); + auto gather = std::make_shared(output, create_const(attr.get_order()), + create_const({attr.get_axis()})); + input.replace_source_output(gather); + // TODO: copy runtime info from RIC sub-graph + }); + ric_attr::set(input, ric_const); + } + + ric_attr::set(m.get_match_value(), ric); + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class Convolution : public ngraph::pass::MatcherPass { +public: + Convolution() { + MATCHER_SCOPE(Convolution); + // Handle Convolution with Constant and FQ on weights. As Convolution is + // a terminal node, so we do not propagate RIC attribute further and insert + // final RIC attribute to the weights input. + auto input_p = pattern::any_input(ric_attr::has>); + auto pattern_root = pattern::wrap_type({input_p, + pattern::wrap_type()}); + auto callback = [=](pattern::Matcher& m) { + auto conv = m.get_match_root(); + auto ric = ric_attr::get(conv->input_value(0)).propagate(); + if (ric.get_axis() != 1) return false; + + ric.set_is_final(true); + ric.set_callback([](Input input, const ric_attr::Attribute & attr) { + auto weights = input.get_source_output(); + auto gather = std::make_shared(weights, create_const(attr.get_order()), + create_const({1} /* output channel */)); + input.replace_source_output(gather); + // TODO: copy runtime info from RIC sub-graph + }); + + if (auto fq = std::dynamic_pointer_cast(conv->get_input_node_shared_ptr(1))) { + // Set final RIC attr to the first FQ input + ric_attr::set(fq->input(0), ric); + + // Apply Binary transformation for FQ to handle 1..5 inputs + ric.set_is_final(false); + ric_attr::set(fq->input_value(0), ric); // set ric attr to simulate propagation flow + Binary().apply(fq); + } else { + ric_attr::set(conv->input(1), ric); + } + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class GroupConvolution : public ngraph::pass::MatcherPass { +public: + GroupConvolution() { + MATCHER_SCOPE(GroupConvolution); + auto input_p = pattern::any_input(ric_attr::has>); + auto pattern_root = pattern::wrap_type({input_p, pattern::wrap_type(pattern::has_static_shape())}); + + auto callback = [=](pattern::Matcher& m) { + auto conv = m.get_match_root(); + const auto & weights_shape = conv->input_value(1).get_shape(); + const int64_t & group = static_cast(weights_shape.at(0)); + const int64_t & channels = static_cast(weights_shape.at(1)); + + auto ric = ric_attr::get(conv->input_value(0)).propagate(); + if (ric.get_order().size() != static_cast(group) || ric.get_axis() != 1) { + // TODO: insert RIC when group == 1 + return false; + } + + // Update weights with RIC attribute + auto ric_weights = ric; + ric_weights.set_is_final(true); + ric_weights.set_axis(0); + ric_weights.set_callback([](Input input, const ric_attr::Attribute & attr) { + auto weights = input.get_source_output(); + auto gather = std::make_shared(weights, create_const(attr.get_order()), + create_const({0} /* output channel */)); + input.replace_source_output(gather); + // TODO: copy runtime info from RIC sub-graph + }); + + if (auto fq = std::dynamic_pointer_cast(conv->get_input_node_shared_ptr(1))) { + // Set final RIC attr to the first FQ input + ric_attr::set(fq->input(0), ric_weights); + + // Apply Binary transformation for FQ to handle 1..5 inputs + ric_weights.set_is_final(false); + ric_attr::set(fq->input_value(0), ric_weights); // set ric attr to simulate propagation flow + Binary().apply(fq); + } else { + ric_attr::set(conv->input(1), ric_weights); + } + + // Calculate new order for RIC propagation + const int64_t output_channels = group * channels; + std::vector new_order; + new_order.reserve(output_channels); + for (const auto & index : ric.get_order()) { + for (int64_t pos = index * channels, i = 0; i < channels; ++i, ++pos) { + new_order.emplace_back(pos); + } + } + assert(new_order.size() == static_cast(output_channels)); + + ric.set_order(new_order); + ric_attr::set(conv->output(0), ric); + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class ShapeOf : public ngraph::pass::MatcherPass { +public: + ShapeOf() { + MATCHER_SCOPE(ShapeOf); + auto pattern_root = pattern::wrap_type(); + + auto callback = [=](pattern::Matcher& m) { + // Skip propagation for ShapeOf path + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class PassThrough : public ngraph::pass::MatcherPass { +public: + PassThrough() { + MATCHER_SCOPE(PassThrough); + auto pattern_root = pattern::wrap_type(); + + auto callback = [=](pattern::Matcher& m) { + auto root = m.get_match_root(); + if (!ric_attr::has(root->input_value(0))) return false; + ric_attr::set(root->output(0), ric_attr::get(root->input_value(0)).propagate()); + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class Transpose : public ngraph::pass::MatcherPass { +public: + Transpose() { + MATCHER_SCOPE(Transpose); + auto input_p = pattern::any_input(ric_attr::has>); + auto order_p = pattern::wrap_type(); + auto pattern_root = pattern::wrap_type({input_p, order_p}); + + auto callback = [=](pattern::Matcher& m) { + const auto & pattern_map = m.get_pattern_value_map(); + auto input = pattern_map.at(input_p); + auto ric = ric_attr::get(input).propagate(); + + auto order_node = std::dynamic_pointer_cast(pattern_map.at(order_p).get_node_shared_ptr()); + auto order = order_node->cast_vector(); + + int64_t new_axis = std::find(order.begin(), order.end(), ric.get_axis()) - order.begin(); + ric.set_axis(new_axis); + + ric_attr::set(m.get_match_value(), ric); + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class Unsupported : public ngraph::pass::MatcherPass { +public: + Unsupported() { + MATCHER_SCOPE(Unsupported); + auto pattern_root = pattern::any_input(); + auto callback = [=](pattern::Matcher& m) { + for (const auto & input : m.get_match_root()->input_values()) { + if (ric_attr::has(input)) { + auto ric = ric_attr::get(input); + ric.set_can_be_fused(false); + NGRAPH_DEBUG << "Node is unsupported by RIC Fusion: " << *m.get_match_root() << std::endl; + } + } + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; +}// namespace prop + +namespace fuse { +namespace { +bool need_to_erase_ric(const Output & output) { + if (!ric_attr::has(output)) return false; + const auto & ric = ric_attr::get(output); + return ric.can_be_fused() && ric.is_initial(); +} +}// namespace + +class InsertReverseInputChannel : public ngraph::pass::MatcherPass { +public: + InsertReverseInputChannel() { + MATCHER_SCOPE(InsertReverseInputChannel); + auto pattern_root = pattern::any_input(); + auto callback = [](pattern::Matcher& m) { + const auto & node = m.get_match_root(); + for (const auto & input : node->inputs()) { + if (!ric_attr::has(input)) continue; + const auto & ric = ric_attr::get(input); + if (ric.can_be_fused() && ric.is_final()) { + ric(input); + } + } + return false; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class EraseSplitConcat : public ngraph::pass::MatcherPass { +public: + EraseSplitConcat() { + MATCHER_SCOPE(EraseSplitConcat); + auto input_p = pattern::any_input(); + auto split_p = pattern::wrap_type({input_p, pattern::any_input()}); + auto pattern_root = pattern::wrap_type({split_p, split_p, split_p}, need_to_erase_ric); + + auto callback = [=](pattern::Matcher& m) { + const auto & pattern_map = m.get_pattern_value_map(); + auto output = pattern_map.at(pattern_root); + auto input = pattern_map.at(input_p); + output.replace(input); + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; + +class EraseGather : public ngraph::pass::MatcherPass { +public: + EraseGather() { + MATCHER_SCOPE(EraseGather); + auto input_p = pattern::any_input(); + auto pattern_root = pattern::wrap_type({input_p, pattern::any_input(), + pattern::any_input()}, + need_to_erase_ric); + auto callback = [=](pattern::Matcher& m) { + const auto & pattern_map = m.get_pattern_value_map(); + auto output = pattern_map.at(pattern_root); + auto input = pattern_map.at(input_p); + output.replace(input); + return true; + }; + + auto m = std::make_shared(pattern_root, matcher_name); + register_matcher(m, callback); + } +}; +}// namespace fuse + +bool ngraph::pass::ReverseInputChannelsFusion::run_on_model(const std::shared_ptr & model) { + Manager m; + m.set_per_pass_validation(false); + + // First we need to initialize and propagate RIC attributes through entire graph + auto ric_prop = m.register_pass(); + ric_prop->add_matcher(); + ric_prop->add_matcher(); + ric_prop->add_matcher(); + ric_prop->add_matcher(); + ric_prop->add_matcher(); + ric_prop->add_matcher(); + ric_prop->add_matcher(); + ric_prop->add_matcher(); + ric_prop->add_matcher(); + + // TODO: validate attributes by request + + // Second we fuse available RIC into nodes and remove original nodes related to fused RIC + auto ric_fuse = m.register_pass(); + ric_fuse->add_matcher(); + ric_fuse->add_matcher(); + ric_fuse->add_matcher(); + + m.run_passes(model); + return false; +} +}// namespace pass +}// namespace ngraph \ No newline at end of file