From 434fcec53cea53a5ef4053c1ad866988c7d43336 Mon Sep 17 00:00:00 2001 From: Ivan Tikhonov Date: Mon, 12 Jul 2021 08:09:41 +0300 Subject: [PATCH] DeformableConvolution v8: MO and IR reader (#6532) * ModulatedDeformableConvolution extractor * Update MO to generate DeformableConv opset8 * clean up * ir reader tests * fix bilinear_interpolation_pad attribute * update BOM file * fix BOM file * delete unnecessary tests * resolve review comments * fix e2e test * fix e2e test * resolve review comments * add comment --- .../src/cldnn_engine/cldnn_engine.cpp | 1 - .../src/mkldnn_plugin/mkldnn_plugin.cpp | 1 + .../src/readers/ir_reader/ie_ir_parser.cpp | 3 + .../common_optimizations.cpp | 2 + .../src/transformations/serialize.cpp | 4 +- model-optimizer/automation/package_BOM.txt | 2 + .../extensions/back/blob_normalizer.py | 2 +- .../extensions/back/op_versioning.py | 1 - .../mxnet/modulated_deformable_conv_ext.py | 61 +++++++++++++++++++ .../modulated_deformable_conv_replacer.py | 22 +++++++ model-optimizer/mo/ops/convolution.py | 3 +- .../mo/ops/deformable_convolution.py | 8 ++- .../ngraph/op/deformable_convolution.hpp | 2 +- 13 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 model-optimizer/extensions/front/mxnet/modulated_deformable_conv_ext.py create mode 100644 model-optimizer/extensions/front/mxnet/modulated_deformable_conv_replacer.py diff --git a/inference-engine/src/cldnn_engine/cldnn_engine.cpp b/inference-engine/src/cldnn_engine/cldnn_engine.cpp index 7849101cc2a..72a34dd855a 100644 --- a/inference-engine/src/cldnn_engine/cldnn_engine.cpp +++ b/inference-engine/src/cldnn_engine/cldnn_engine.cpp @@ -192,7 +192,6 @@ InferenceEngine::CNNNetwork clDNNEngine::CloneAndTransformNetwork(const Inferenc manager.register_pass(); manager.register_pass(); manager.register_pass(); - manager.register_pass(); static const precisions_array convert_precision_list { {ngraph::element::i64, ngraph::element::i32}, diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp index 3f53db150ce..733d785d594 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp @@ -56,6 +56,7 @@ #include #include #include +#include #include #include #include diff --git a/inference-engine/src/readers/ir_reader/ie_ir_parser.cpp b/inference-engine/src/readers/ir_reader/ie_ir_parser.cpp index 892c2e6bae4..cc00ec77114 100644 --- a/inference-engine/src/readers/ir_reader/ie_ir_parser.cpp +++ b/inference-engine/src/readers/ir_reader/ie_ir_parser.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -838,6 +839,7 @@ std::shared_ptr XmlDeserializer::createNode( } ngraphNode->set_arguments(inputs); XmlDeserializer visitor(node, weights, opsets, variables); + if (ngraphNode->visit_attributes(visitor)) { ngraphNode->constructor_validate_and_infer_types(); } @@ -900,6 +902,7 @@ V10Parser::V10Parser(const std::vector& exts) : _exts(exts) { opsets["opset5"] = ngraph::get_opset5(); opsets["opset6"] = ngraph::get_opset6(); opsets["opset7"] = ngraph::get_opset7(); + opsets["opset8"] = ngraph::get_opset8(); // Load custom opsets for (const auto& ext : exts) { diff --git a/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp b/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp index 956904c9a34..415ecb11610 100644 --- a/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp +++ b/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp @@ -72,6 +72,7 @@ #include "transformations/op_conversions/mvn6_decomposition.hpp" #include "transformations/op_conversions/simplify_ctc_greedy_decoder_seq_len.hpp" #include "transformations/op_conversions/gather_normalize_negative_indices.hpp" +#include "transformations/op_conversions/convert_deformable_conv_v8_to_v1.hpp" #include #include @@ -180,6 +181,7 @@ bool ngraph::pass::CommonOptimizations::run_on_function(std::shared_ptr(); manager.register_pass(); manager.register_pass(); + manager.register_pass(); auto fq_fusions = manager.register_pass(); fq_fusions->add_matcher(); diff --git a/inference-engine/src/transformations/src/transformations/serialize.cpp b/inference-engine/src/transformations/src/transformations/serialize.cpp index 73b72313e5f..1fd41881125 100644 --- a/inference-engine/src/transformations/src/transformations/serialize.cpp +++ b/inference-engine/src/transformations/src/transformations/serialize.cpp @@ -465,10 +465,10 @@ const std::vector create_edge_mapping( std::string get_opset_name( const ngraph::Node* n, const std::map& custom_opsets) { - auto opsets = std::array, 7>{ + auto opsets = std::array, 8>{ ngraph::get_opset1(), ngraph::get_opset2(), ngraph::get_opset3(), ngraph::get_opset4(), ngraph::get_opset5(), ngraph::get_opset6(), - ngraph::get_opset7()}; + ngraph::get_opset7(), ngraph::get_opset8()}; auto special_opset = get_special_opset_for_op(n->get_type_info()); if (!special_opset.empty()) { diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 671c45fd68b..4adb1e22aa2 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -212,6 +212,8 @@ extensions/front/mxnet/instance_norm_ext.py extensions/front/mxnet/leaky_relu.py extensions/front/mxnet/lrn_ext.py extensions/front/mxnet/max_ext.py +extensions/front/mxnet/modulated_deformable_conv_ext.py +extensions/front/mxnet/modulated_deformable_conv_replacer.py extensions/front/mxnet/multibox_detection_ext.py extensions/front/mxnet/mx_reshape_reverse.py extensions/front/mxnet/mx_reshape_to_reshape.py diff --git a/model-optimizer/extensions/back/blob_normalizer.py b/model-optimizer/extensions/back/blob_normalizer.py index 53f1118eac5..4c35de5634e 100644 --- a/model-optimizer/extensions/back/blob_normalizer.py +++ b/model-optimizer/extensions/back/blob_normalizer.py @@ -46,7 +46,7 @@ class BlobNormalizer(BackReplacementPattern): def find_and_replace_pattern(self, graph: Graph): for node in graph.get_op_nodes(): if node.soft_get('type').lower() not in OpVersioning.opset_1_types and \ - not node.soft_get('version') in ["opset2", "opset3", "opset4"]: + not node.soft_get('version') in ["opset2", "opset3", "opset4", "opset8"]: continue for _, d in node.in_edges().items(): diff --git a/model-optimizer/extensions/back/op_versioning.py b/model-optimizer/extensions/back/op_versioning.py index f8a8141a8c0..5afd10a5d12 100644 --- a/model-optimizer/extensions/back/op_versioning.py +++ b/model-optimizer/extensions/back/op_versioning.py @@ -33,7 +33,6 @@ class OpVersioning(BackReplacementPattern): "ConvolutionBackpropData", "Cos", "Cosh", - "DeformableConvolution", "DeformablePSROIPooling", "DepthToSpace", "DetectionOutput", diff --git a/model-optimizer/extensions/front/mxnet/modulated_deformable_conv_ext.py b/model-optimizer/extensions/front/mxnet/modulated_deformable_conv_ext.py new file mode 100644 index 00000000000..cee2d111c98 --- /dev/null +++ b/model-optimizer/extensions/front/mxnet/modulated_deformable_conv_ext.py @@ -0,0 +1,61 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np + +from mo.front.extractor import FrontExtractorOp +from mo.front.mxnet.extractors.utils import get_mxnet_layer_attrs +from mo.ops.deformable_convolution import DeformableConvolution + + +class ModulatedDeformableConvolutionExtractor(FrontExtractorOp): + op = '_contrib_ModulatedDeformableConvolution' + enabled = True + + @classmethod + def extract(cls, node): + attr = get_mxnet_layer_attrs(node.symbol_dict) + + kernel = attr.tuple("kernel", int, None) + stride = attr.tuple("stride", int, tuple(np.ones(len(kernel), dtype=np.int64))) + padding = attr.tuple("pad", int, tuple(np.zeros(len(kernel), dtype=np.int64))) + dilate = attr.tuple("dilate", int, tuple(np.ones(len(kernel), dtype=np.int64))) + num_deformable_group = attr.int("num_deformable_group", 1) + num_group = attr.int("num_group", 1) + output = attr.int("num_filter", None) + bias_term = attr.str("no_bias", 'False') == 'False' + + final_dilations = np.array([1, 1, *[d for d in dilate]], dtype=np.int64) if dilate is not None else None + + node_attrs = { + 'op': __class__.op, + 'bias_addable': True, + 'bias_term': bias_term, + 'pad': np.array([[0, 0], [0, 0], *[[pad, pad] for pad in padding]], dtype=np.int64), + 'pad_spatial_shape': np.array([[pad, pad] for pad in padding], dtype=np.int64), + 'dilation': final_dilations, + 'output_spatial_shape': None, + 'output_shape': None, + 'stride': np.array([1, 1, *[s for s in stride]], dtype=np.int64), + 'group': num_group, + 'deformable_group': num_deformable_group, + 'output': output, + 'kernel_spatial': np.array([k for k in kernel], dtype=np.int64), + 'bilinear_interpolation_pad': True, + + 'input_feature_channel': 1, + 'output_feature_channel': 0, + 'kernel_spatial_idx': None, + 'reshape_kernel': True, + 'weights_index': 2, + 'in_ports_count': 4, + + 'spatial_dims': None, + 'channel_dims': np.array([1], dtype=np.int64), + 'batch_dims': np.array([0], dtype=np.int64), + 'layout': 'NCHW', + } + + # update the attributes of the node + DeformableConvolution.update_node_stat(node, node_attrs) + return cls.enabled diff --git a/model-optimizer/extensions/front/mxnet/modulated_deformable_conv_replacer.py b/model-optimizer/extensions/front/mxnet/modulated_deformable_conv_replacer.py new file mode 100644 index 00000000000..e3f95fc0f5a --- /dev/null +++ b/model-optimizer/extensions/front/mxnet/modulated_deformable_conv_replacer.py @@ -0,0 +1,22 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from mo.front.common.replacement import FrontReplacementPattern +from mo.graph.graph import Graph + + +class DeformableConvolutionReplacer(FrontReplacementPattern): + # swap mask and weights inputs for ModulatedDeformableConvolution according to the specification + enabled = True + + def find_and_replace_pattern(self, graph: Graph): + + for deform_conv in graph.get_op_nodes(type='DeformableConvolution'): + if len(deform_conv.get_inputs()) != 4: + return + + m_source = deform_conv.in_port(2).get_source() + deform_conv.in_port(2).disconnect() + + deform_conv.in_port(3).get_connection().set_destination(deform_conv.in_port(2)) + m_source.connect(deform_conv.in_port(3)) diff --git a/model-optimizer/mo/ops/convolution.py b/model-optimizer/mo/ops/convolution.py index aa884b34721..ee24fcd1370 100644 --- a/model-optimizer/mo/ops/convolution.py +++ b/model-optimizer/mo/ops/convolution.py @@ -102,7 +102,6 @@ class Convolution(Op): node['bias_term'] = len(node.in_nodes()) == 3 weights_index = node.weights_index if node.has_valid('weights_index') else 1 - # Reshape weights kernel to original shape # In case of caffe or MXNet framework, values for weights have no structured shape like OIHW # so we have to reshape weights to normal shape @@ -237,6 +236,8 @@ class Convolution(Op): for n in node.out_nodes(): node.out_node(n).shape = output_shape + # bin attribute is used for pre-processing, but it will be deleted in BlobNormalizer transformation + # and the blobs (weights, biases) will be represented as inputs to the node mark_input_bins(node, start_port=1 if node.type != 'DeformableConvolution' else 2) assign_dims_to_weights(node.in_node(weights_index), node.kernel_spatial_idx, node.input_feature_channel, node.output_feature_channel, len(kernel_shape)) diff --git a/model-optimizer/mo/ops/deformable_convolution.py b/model-optimizer/mo/ops/deformable_convolution.py index f219208cbc0..5eb573fb214 100644 --- a/model-optimizer/mo/ops/deformable_convolution.py +++ b/model-optimizer/mo/ops/deformable_convolution.py @@ -13,7 +13,7 @@ class DeformableConvolution(Op): super().__init__(graph, { 'type': __class__.op, 'op': __class__.op, - 'version': 'opset1', + 'version': 'opset8', 'infer': Convolution.infer, 'group': 1, 'deformable_group': 1, @@ -21,8 +21,10 @@ class DeformableConvolution(Op): 'multiplication_transparent_ports': [(0, 0), (2, 0)], 'in_ports_count': 3, 'out_ports_count': 1, + 'bilinear_interpolation_pad': False, }, attrs) def backend_attrs(self): - # the same attributes as in a regular convolution and one additional attribute 'deformable_group' and 'group' - return Convolution(self.graph, {}).backend_attrs() + ['deformable_group', 'group'] + # the same attributes as in a regular convolution and additional attributes 'deformable_group', 'group' + # and 'bilinear_interpolation_pad' + return Convolution(self.graph, {}).backend_attrs() + ['deformable_group', 'group', 'bilinear_interpolation_pad'] diff --git a/ngraph/core/include/ngraph/op/deformable_convolution.hpp b/ngraph/core/include/ngraph/op/deformable_convolution.hpp index f55535cb28f..da6b18c22b5 100644 --- a/ngraph/core/include/ngraph/op/deformable_convolution.hpp +++ b/ngraph/core/include/ngraph/op/deformable_convolution.hpp @@ -171,7 +171,7 @@ namespace ngraph } private: - int64_t m_bilinear_interpolation_pad; + bool m_bilinear_interpolation_pad; }; } // namespace v8 } // namespace op