From e3270b6b34271ef6a788a2f9d848bba5b0b9412a Mon Sep 17 00:00:00 2001 From: Gleb Kazantaev Date: Thu, 1 Oct 2020 20:08:41 +0300 Subject: [PATCH] Move passes to CommonOptimizations (#2442) * Move passes to CommonOptimizations * Updated BN tests to use ranges for constant value generation * Added some decomposition passes into legacy conversion * Added WA for FQReshapeFusion pass --- .../include/transformations/convert_gelu.hpp | 9 +-- .../common_optimizations.cpp | 66 +++++++++++++++++-- .../fq_reshape_fusion.cpp | 9 ++- .../convert_broadcast_to_tiles.cpp | 4 +- .../convert_opset1_to_legacy/convert_gelu.cpp | 12 ++-- .../convert_opset1_to_legacy.cpp | 51 ++++---------- .../convert_opset2_to_opset1.cpp | 1 - .../transformations/hswish_decomposition.cpp | 2 +- .../reduce_l1_decomposition.cpp | 2 +- .../reduce_l2_decomposition.cpp | 2 +- .../transformations/fq_reshape_fusion.cpp | 46 +++++++++++++ .../tests/ngraph_functions/src/batch_norm.cpp | 6 +- 12 files changed, 143 insertions(+), 67 deletions(-) diff --git a/inference-engine/src/transformations/include/transformations/convert_gelu.hpp b/inference-engine/src/transformations/include/transformations/convert_gelu.hpp index 5ee3c81790c..0057699c99c 100644 --- a/inference-engine/src/transformations/include/transformations/convert_gelu.hpp +++ b/inference-engine/src/transformations/include/transformations/convert_gelu.hpp @@ -21,13 +21,8 @@ class TRANSFORMATIONS_API ConvertGELU; } // namespace pass } // namespace ngraph -class ngraph::pass::ConvertGELU: public ngraph::pass::GraphRewrite { +class ngraph::pass::ConvertGELU: public ngraph::pass::MatcherPass { public: NGRAPH_RTTI_DECLARATION; - ConvertGELU() : GraphRewrite() { - convert_gelu(); - } - -private: - void convert_gelu(); + ConvertGELU(); }; diff --git a/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp b/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp index e45776a9c78..fae8de5cfcb 100644 --- a/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp +++ b/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp @@ -7,21 +7,40 @@ #include "transformations/common_optimizations/algebraic_simplification.hpp" #include "transformations/common_optimizations/nop_elimination.hpp" #include "transformations/common_optimizations/common_optimizations.hpp" +#include "transformations/common_optimizations/conv_mul_fusion.hpp" +#include "transformations/common_optimizations/fq_mul_fusion.hpp" +#include "transformations/common_optimizations/fq_reshape_fusion.hpp" #include "transformations/depth_to_space_fusion.hpp" #include "transformations/optimize_strided_slice.hpp" -#include "transformations/convert_scatter_elements_to_scatter.hpp" -#include "transformations/convert_pad_to_group_conv.hpp" -#include "transformations/remove_filtering_boxes_by_size.hpp" #include "transformations/init_node_info.hpp" #include "transformations/itt.hpp" #include "transformations/mish_fusion.hpp" #include "transformations/softplus_fusion.hpp" #include "transformations/softplus_to_mish_fusion.hpp" #include "transformations/swish_fusion.hpp" -#include "transformations/hswish_fusion.hpp" #include "transformations/normalize_l2_fusion.hpp" -#include "transformations/convert_quantize_dequantize.hpp" #include "transformations/bidirectional_sequences_decomposition.hpp" +#include "transformations/convert_pad_to_group_conv.hpp" +#include "transformations/convert_divide.hpp" +#include "transformations/convert_quantize_dequantize.hpp" +#include "transformations/convert_mod.hpp" +#include "transformations/convert_minimum_to_power_and_max.hpp" +#include "transformations/convert_negative.hpp" +#include "transformations/convert_scatter_elements_to_scatter.hpp" +#include "transformations/convert_reduce_to_pooling.hpp" +#include "transformations/convert_subtract.hpp" +#include "transformations/convert_depth_to_space.hpp" +#include "transformations/convert_space_to_depth.hpp" +#include "transformations/convert_broadcast_to_tiles.hpp" +#include "transformations/convert_gelu.hpp" +#include "transformations/batch_norm_decomposition.hpp" +#include "transformations/pull_transpose_through_fq.hpp" +#include "transformations/lin_op_sequence_fusoin.hpp" +#include "transformations/reduce_l1_decomposition.hpp" +#include "transformations/reduce_l2_decomposition.hpp" +#include "transformations/remove_filtering_boxes_by_size.hpp" +#include "transformations/hswish_decomposition.hpp" +#include "transformations/hswish_fusion.hpp" #include #include @@ -55,6 +74,43 @@ bool ngraph::pass::CommonOptimizations::run_on_function(std::shared_ptr(); manager.register_pass(); + auto decomp = manager.register_pass(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->add_matcher(); + decomp->set_name("ngraph::pass::CommonDecompositions"); + + // CF is required after all decompositions + manager.register_pass(); + + // LinOpSequenceFusion must be executed after all decompositions + manager.register_pass(); + + manager.register_pass(); + manager.register_pass(); + manager.register_pass(); + manager.register_pass(); + manager.register_pass(); + + auto fq_fusions = manager.register_pass(); + fq_fusions->add_matcher(); + fq_fusions->add_matcher(); + fq_fusions->add_matcher(); + fq_fusions->set_name("ngraph::pass::FakeQuantizeFusions"); + manager.set_callback(m_transformation_callback); manager.run_passes(f); return true; diff --git a/inference-engine/src/transformations/src/transformations/common_optimizations/fq_reshape_fusion.cpp b/inference-engine/src/transformations/src/transformations/common_optimizations/fq_reshape_fusion.cpp index 0f7127ca275..32ecf1c66b3 100644 --- a/inference-engine/src/transformations/src/transformations/common_optimizations/fq_reshape_fusion.cpp +++ b/inference-engine/src/transformations/src/transformations/common_optimizations/fq_reshape_fusion.cpp @@ -20,7 +20,14 @@ ngraph::pass::FakeQuantizeReshapeFusion::FakeQuantizeReshapeFusion() { ngraph::pattern::any_input()}, pattern::consumers_count(1)); const auto reshape_node_p = ngraph::pattern::wrap_type( - {fq_node_p, ngraph::pattern::any_input()}); + {fq_node_p, ngraph::pattern::any_input()}, [](const Output & output) { + // WA: check that all Reshape node consumers are not GroupConvolution operations + const auto & target_inputs = output.get_target_inputs(); + return std::all_of(target_inputs.begin(), target_inputs.end(), + [](const Input & input){ + return input.get_node()->get_type_info() != opset4::GroupConvolution::type_info; + }); + }); ngraph::matcher_pass_callback callback = [=](pattern::Matcher &m) { const auto &pattern_map = m.get_pattern_value_map(); diff --git a/inference-engine/src/transformations/src/transformations/convert_broadcast_to_tiles.cpp b/inference-engine/src/transformations/src/transformations/convert_broadcast_to_tiles.cpp index 895a0e94af0..939648cf2aa 100644 --- a/inference-engine/src/transformations/src/transformations/convert_broadcast_to_tiles.cpp +++ b/inference-engine/src/transformations/src/transformations/convert_broadcast_to_tiles.cpp @@ -16,7 +16,7 @@ NGRAPH_RTTI_DEFINITION(ngraph::pass::ConvertBroadcastToTiles, "ConvertBroadcastT ngraph::pass::ConvertBroadcastToTiles::ConvertBroadcastToTiles() { auto broadcast = ngraph::pattern::wrap_type(); - ngraph::graph_rewrite_callback callback = [](pattern::Matcher& m) { + ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { auto broadcast = std::dynamic_pointer_cast(m.get_match_root()); if (!broadcast) { @@ -83,7 +83,7 @@ ngraph::pass::ConvertBroadcastToTiles::ConvertBroadcastToTiles() { } auto const_node = std::make_shared(element::i64, Shape {dims_count}, dims); - auto tile = std::make_shared(last_node, const_node); + auto tile = register_new_node(last_node, const_node); new_ops.push_back(tile); tile->set_friendly_name(broadcast->get_friendly_name()); diff --git a/inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_gelu.cpp b/inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_gelu.cpp index a69be94aa4d..2a9fc54f4fa 100644 --- a/inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_gelu.cpp +++ b/inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_gelu.cpp @@ -12,7 +12,7 @@ NGRAPH_RTTI_DEFINITION(ngraph::pass::ConvertGELU, "ConvertGELU", 0); -void ngraph::pass::ConvertGELU::convert_gelu() { +ngraph::pass::ConvertGELU::ConvertGELU() { auto input = std::make_shared(element::f32, Shape{}); auto gelu = std::make_shared(input); @@ -24,11 +24,11 @@ void ngraph::pass::ConvertGELU::convert_gelu() { auto input_type = input.get_element_type(); // f(x) = 0.5 * x * (1.0 + erf( x / sqrt(2.0) ) - auto mul = std::make_shared(input, ngraph::op::Constant::create(input_type, Shape{}, {0.5})); - auto sq2 = std::make_shared(ngraph::op::Constant::create(input_type, Shape{}, {2.0})); - auto div = std::make_shared(input, sq2); + auto mul = std::make_shared(input, ngraph::opset1::Constant::create(input_type, Shape{}, {0.5})); + auto sq2 = std::make_shared(ngraph::opset1::Constant::create(input_type, Shape{}, {2.0})); + auto div = register_new_node(input, sq2); // can be decomposed auto erf = std::make_shared(div); - auto add = std::make_shared(erf, ngraph::op::Constant::create(input_type, Shape{}, {1.0})); + auto add = std::make_shared(erf, ngraph::opset1::Constant::create(input_type, Shape{}, {1.0})); auto res = std::make_shared(mul, add); res->set_friendly_name(gelu->get_friendly_name()); @@ -38,5 +38,5 @@ void ngraph::pass::ConvertGELU::convert_gelu() { }; auto m = std::make_shared(gelu, "ConvertGELU"); - this->add_matcher(m, callback, PassProperty::CHANGE_DYNAMIC_STATE); + register_matcher(m, callback, PassProperty::CHANGE_DYNAMIC_STATE); } diff --git a/inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.cpp b/inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.cpp index 82a2e5ce6ec..b6784268879 100644 --- a/inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.cpp +++ b/inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.cpp @@ -68,48 +68,26 @@ bool ngraph::pass::ConvertOpSet1ToLegacy::run_on_function(std::shared_ptr > transforms; manager.register_pass(); - // the following two transformations produce ReduceSum operations so they - // must be executed before the ConvertReduceSumToPooling transformation - manager.register_pass(); - manager.register_pass(); - - // HSwishDecomposition produce Minimum, Relu and Multiply operations - // so it must be executed before - manager.register_pass(); - - // List if Decomposition and Conversion transformations that can be - // applied simultaneously in a single graph traversal + // Some passes before ConvertOpSet1ToLegacy can produce some of this + // operations. So for convenience we decompose this operations here and + // in CommonOptimizations. auto decomp = manager.register_pass(); - decomp->add_matcher(); - decomp->add_matcher(); - decomp->add_matcher(); - decomp->add_matcher(); decomp->add_matcher(); decomp->add_matcher(); decomp->add_matcher(); decomp->add_matcher(); decomp->add_matcher(); - decomp->add_matcher(); - decomp->add_matcher(); - decomp->add_matcher(); - decomp->add_matcher(); - decomp->add_matcher(); - decomp->set_name("ngraph::pass::Decompositions"); + decomp->set_name("ngraph::pass::LegacyDecompositions"); - // CF is required after all decompositions - manager.register_pass(); + auto convert_matmul = manager.register_pass(); + convert_matmul->add_matcher(); + convert_matmul->add_matcher(); + convert_matmul->add_matcher(); + convert_matmul->set_name("ngraph::pass::ConvertMatMul"); - // LinOpSequenceFusion must be executed after all decompositions - manager.register_pass(); - - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); manager.register_pass(); // Convolution/Deconvolution/FullyConnected fusions @@ -120,18 +98,12 @@ bool ngraph::pass::ConvertOpSet1ToLegacy::run_on_function(std::shared_ptradd_matcher(); convert_convolutions->set_name("ngraph::pass::ConvertConvolutions"); - auto fq_fusions = manager.register_pass(); - fq_fusions->add_matcher(); - fq_fusions->add_matcher(); - fq_fusions->add_matcher(); - fq_fusions->set_name("ngraph::pass::FakeQuantizeFusions"); - // Convolution/Deconvolution/FullyConnected fusions auto fusion = manager.register_pass(); fusion->add_matcher(); fusion->add_matcher(); fusion->add_matcher(); - fusion->set_name("ngraph::pass::Fusions"); + fusion->set_name("ngraph::pass::BiasFusions"); // CF is required after fusions manager.register_pass(); @@ -148,6 +120,7 @@ bool ngraph::pass::ConvertOpSet1ToLegacy::run_on_function(std::shared_ptradd_matcher(); anchor->add_matcher(); anchor->add_matcher(); + anchor->add_matcher(); anchor->add_matcher(); anchor->add_matcher(); anchor->add_matcher(); @@ -170,7 +143,7 @@ bool ngraph::pass::ConvertOpSet1ToLegacy::run_on_function(std::shared_ptradd_matcher(); anchor->add_matcher(); anchor->add_matcher(); - anchor->set_name("ngraph::pass::ConvertOpSet1ToLegacy"); + anchor->set_name("ngraph::pass::LegacyConversions"); // List of final conversion transformations that must to be executed // after previous group of transformations diff --git a/inference-engine/src/transformations/src/transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.cpp b/inference-engine/src/transformations/src/transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.cpp index 93588e359da..80b66b3ac0d 100644 --- a/inference-engine/src/transformations/src/transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.cpp +++ b/inference-engine/src/transformations/src/transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.cpp @@ -21,7 +21,6 @@ bool ngraph::pass::ConvertOpSet2ToOpSet1::run_on_function(std::shared_ptr(); manager.register_pass(); manager.register_pass(); diff --git a/inference-engine/src/transformations/src/transformations/hswish_decomposition.cpp b/inference-engine/src/transformations/src/transformations/hswish_decomposition.cpp index 2da0f618c4b..e0a52fbd72a 100644 --- a/inference-engine/src/transformations/src/transformations/hswish_decomposition.cpp +++ b/inference-engine/src/transformations/src/transformations/hswish_decomposition.cpp @@ -29,7 +29,7 @@ ngraph::pass::HSwishDecomposition::HSwishDecomposition() { auto add = std::make_shared(hswish_node->input_value(0), add_constant); auto relu = std::make_shared(add); auto min_constant = ngraph::opset4::Constant::create(input_type, ngraph::Shape{}, {6.0}); - auto min = std::make_shared(relu, min_constant); + auto min = register_new_node(relu, min_constant); auto mul_first = std::make_shared(hswish_node->input_value(0), min); auto mul_constant = ngraph::opset4::Constant::create(input_type, ngraph::Shape{}, {(1.0/6.0)}); // const(1/6) auto mul_second = std::make_shared(mul_first, mul_constant); diff --git a/inference-engine/src/transformations/src/transformations/reduce_l1_decomposition.cpp b/inference-engine/src/transformations/src/transformations/reduce_l1_decomposition.cpp index 87f8fd0481b..6d83866a4e3 100644 --- a/inference-engine/src/transformations/src/transformations/reduce_l1_decomposition.cpp +++ b/inference-engine/src/transformations/src/transformations/reduce_l1_decomposition.cpp @@ -25,7 +25,7 @@ ngraph::pass::ReduceL1Decomposition::ReduceL1Decomposition() { } auto abs = std::make_shared(reduce_l1_node->input_value(0)); - auto reduce_sum = std::make_shared(abs, reduce_l1_node->input_value(1), reduce_l1_node->get_keep_dims()); + auto reduce_sum = register_new_node(abs, reduce_l1_node->input_value(1), reduce_l1_node->get_keep_dims()); reduce_sum->set_friendly_name(m.get_match_root()->get_friendly_name()); ngraph::copy_runtime_info(reduce_l1_node, diff --git a/inference-engine/src/transformations/src/transformations/reduce_l2_decomposition.cpp b/inference-engine/src/transformations/src/transformations/reduce_l2_decomposition.cpp index f36cfeef67c..87c58b5c3f2 100644 --- a/inference-engine/src/transformations/src/transformations/reduce_l2_decomposition.cpp +++ b/inference-engine/src/transformations/src/transformations/reduce_l2_decomposition.cpp @@ -26,7 +26,7 @@ ngraph::pass::ReduceL2Decomposition::ReduceL2Decomposition() { auto const_2 = ngraph::opset4::Constant::create(reduce_l2_node->input_value(0).get_element_type(), Shape{}, {2.0f}); auto square = std::make_shared(reduce_l2_node->input_value(0), const_2); - auto reduce_sum = std::make_shared(square, reduce_l2_node->input_value(1), reduce_l2_node->get_keep_dims()); + auto reduce_sum = register_new_node(square, reduce_l2_node->input_value(1), reduce_l2_node->get_keep_dims()); auto sqrt = std::make_shared(reduce_sum); reduce_sum->set_friendly_name(m.get_match_root()->get_friendly_name()); ngraph::copy_runtime_info(reduce_l2_node, diff --git a/inference-engine/tests/functional/inference_engine/transformations/fq_reshape_fusion.cpp b/inference-engine/tests/functional/inference_engine/transformations/fq_reshape_fusion.cpp index aea06e15ace..8f166131c1a 100644 --- a/inference-engine/tests/functional/inference_engine/transformations/fq_reshape_fusion.cpp +++ b/inference-engine/tests/functional/inference_engine/transformations/fq_reshape_fusion.cpp @@ -123,3 +123,49 @@ INSTANTIATE_TEST_CASE_P(NGraph, nGraphFQReshapeFusionTests, testing::Values( FQReshapeFusionTestCase{{1, 2, 1, 3}, {2, 1, 3}, {1}, {1, 1}, {1, 2, 1, 1}, {1, 2, 1, 3}, {}, {}, {}, {}, true}, FQReshapeFusionTestCase{{1, 2, 1, 3}, {2, 1, 1}, {1}, {1, 1}, {1, 2, 1, 1}, {6}, {}, {}, {}, {}, true})); } // namespace + +TEST(nGraphFQReshapeFusionTests, FQReshapeGroupConvolution) { + auto get_function = [](const FQReshapeFusionTestCase & test_case) { + const auto & data = std::make_shared(ngraph::element::f32, test_case.data_shape, 0); + auto il = std::make_shared(ngraph::element::f32, test_case.il_shape); + auto ih = std::make_shared(ngraph::element::f32, test_case.ih_shape); + auto ol = std::make_shared(ngraph::element::f32, test_case.ol_shape); + auto oh = std::make_shared(ngraph::element::f32, test_case.oh_shape); + + auto fq = std::make_shared(data, il, ih, ol, oh, 42); + + auto reshape_pattern = std::make_shared( + ngraph::element::i64, ngraph::Shape{test_case.reshape_pattern.size()}, test_case.reshape_pattern); + auto reshape = std::make_shared(fq, reshape_pattern, true); + + auto input = std::make_shared(ngraph::element::f32, test_case.data_shape); + ngraph::Strides stride{1, 1}; + ngraph::CoordinateDiff pad{0, 0}; + auto group_conv = std::make_shared(input, reshape, stride, pad, pad, stride); + + auto result = std::make_shared(group_conv); + ngraph::ParameterVector params = {il, ih, ol, oh, input}; + ngraph::ResultVector results = {result}; + return std::make_shared(results, params); + }; + + FQReshapeFusionTestCase params; + params.data_shape = {1, 2, 1, 3}; + params.il_shape = {2, 1, 1}; + params.ih_shape = {1}; + params.ol_shape = {1, 1}; + params.oh_shape = {1, 2, 1, 1}; + params.reshape_pattern = {2, 3, 1, 1, 1}; + + auto f = get_function(params); + + ngraph::pass::Manager manager; + manager.register_pass(); + manager.register_pass(); + manager.run_passes(f); + + ASSERT_NO_THROW(check_rt_info(f)); + + auto res = compare_functions(f, get_function(params)); + ASSERT_TRUE(res.first) << res.second; +} \ No newline at end of file diff --git a/inference-engine/tests/ngraph_functions/src/batch_norm.cpp b/inference-engine/tests/ngraph_functions/src/batch_norm.cpp index c0f960663d1..14f4035e9e4 100644 --- a/inference-engine/tests/ngraph_functions/src/batch_norm.cpp +++ b/inference-engine/tests/ngraph_functions/src/batch_norm.cpp @@ -15,9 +15,9 @@ std::shared_ptr makeBatchNormInference(const ngraph::Output& size_t C = data.get_shape().at(1); bool random = true; std::vector values(C); - auto gamma = ngraph::builder::makeConstant(ngPrc, ngraph::Shape{C}, values, random); - auto beta = ngraph::builder::makeConstant(ngPrc, ngraph::Shape{C}, values, random); - auto mean = ngraph::builder::makeConstant(ngPrc, ngraph::Shape{C}, values, random); + auto gamma = ngraph::builder::makeConstant(ngPrc, ngraph::Shape{C}, values, random, 1, 0); + auto beta = ngraph::builder::makeConstant(ngPrc, ngraph::Shape{C}, values, random, 1, 0); + auto mean = ngraph::builder::makeConstant(ngPrc, ngraph::Shape{C}, values, random, 1, 0); // Fill the vector for variance with positive values std::default_random_engine gen;