DeformableConvolution v8: MO and IR reader (#6532)

* ModulatedDeformableConvolution extractor

* Update MO to generate DeformableConv opset8

* clean up

* ir reader tests

* fix bilinear_interpolation_pad attribute

* update BOM file

* fix BOM file

* delete unnecessary tests

* resolve review comments

* fix e2e test

* fix e2e test

* resolve review comments

* add comment

inference-engine/src/cldnn_engine/cldnn_engine.cpp

@@ -192,7 +192,6 @@ InferenceEngine::CNNNetwork clDNNEngine::CloneAndTransformNetwork(const Inferenc
             manager.register_pass<ngraph::pass::ConvertNMS4ToNMS5>();
             manager.register_pass<ngraph::pass::ConvertNMSToNMSIEInternal>();
             manager.register_pass<ngraph::pass::ConvertGather0D>();
-            manager.register_pass<ngraph::pass::ConvertDeformableConv8To1>();
 
             static const precisions_array convert_precision_list {
                     {ngraph::element::i64, ngraph::element::i32},

inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp

@@ -56,6 +56,7 @@
 #include <transformations/op_conversions/simplify_ctc_greedy_decoder_seq_len.hpp>
 #include <transformations/op_conversions/convert_previous_nms_to_nms_5.hpp>
 #include <transformations/op_conversions/convert_nms_to_nms_ie_internal.hpp>
+#include <transformations/op_conversions/convert_deformable_conv_v8_to_v1.hpp>
 #include <transformations/convert_precision.hpp>
 #include <transformations/init_node_info.hpp>
 #include <transformations/rt_info/fused_names_attribute.hpp>

inference-engine/src/readers/ir_reader/ie_ir_parser.cpp

@@ -18,6 +18,7 @@
 #include <ngraph/opsets/opset5.hpp>
 #include <ngraph/opsets/opset6.hpp>
 #include <ngraph/opsets/opset7.hpp>
+#include <ngraph/opsets/opset8.hpp>
 #include <ngraph/variant.hpp>
 #include <ngraph_ops/framework_node.hpp>
 #include <set>
@@ -838,6 +839,7 @@ std::shared_ptr<ngraph::Node> XmlDeserializer::createNode(
         }
         ngraphNode->set_arguments(inputs);
         XmlDeserializer visitor(node, weights, opsets, variables);
+
         if (ngraphNode->visit_attributes(visitor)) {
             ngraphNode->constructor_validate_and_infer_types();
         }
@@ -900,6 +902,7 @@ V10Parser::V10Parser(const std::vector<IExtensionPtr>& exts) : _exts(exts) {
     opsets["opset5"] = ngraph::get_opset5();
     opsets["opset6"] = ngraph::get_opset6();
     opsets["opset7"] = ngraph::get_opset7();
+    opsets["opset8"] = ngraph::get_opset8();
 
     // Load custom opsets
     for (const auto& ext : exts) {

inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp

@@ -72,6 +72,7 @@
 #include "transformations/op_conversions/mvn6_decomposition.hpp"
 #include "transformations/op_conversions/simplify_ctc_greedy_decoder_seq_len.hpp"
 #include "transformations/op_conversions/gather_normalize_negative_indices.hpp"
+#include "transformations/op_conversions/convert_deformable_conv_v8_to_v1.hpp"
 
 #include <ngraph/pass/manager.hpp>
 #include <ngraph/pass/constant_folding.hpp>
@@ -180,6 +181,7 @@ bool ngraph::pass::CommonOptimizations::run_on_function(std::shared_ptr<ngraph::
     manager.register_pass<ngraph::pass::ConstantFolding>();
     manager.register_pass<ngraph::pass::ConvertGather7ToGather1>();
     manager.register_pass<ngraph::pass::ConvertGather1ToGather7, false>();
+    manager.register_pass<ngraph::pass::ConvertDeformableConv8To1>();
 
     auto fq_fusions = manager.register_pass<ngraph::pass::GraphRewrite>();
     fq_fusions->add_matcher<ngraph::pass::FakeQuantizeMulFusion>();

inference-engine/src/transformations/src/transformations/serialize.cpp

@@ -465,10 +465,10 @@ const std::vector<Edge> create_edge_mapping(
 std::string get_opset_name(
     const ngraph::Node* n,
     const std::map<std::string, ngraph::OpSet>& custom_opsets) {
-    auto opsets = std::array<std::reference_wrapper<const ngraph::OpSet>, 7>{
+    auto opsets = std::array<std::reference_wrapper<const ngraph::OpSet>, 8>{
         ngraph::get_opset1(), ngraph::get_opset2(), ngraph::get_opset3(),
         ngraph::get_opset4(), ngraph::get_opset5(), ngraph::get_opset6(),
-        ngraph::get_opset7()};
+        ngraph::get_opset7(), ngraph::get_opset8()};
 
     auto special_opset = get_special_opset_for_op(n->get_type_info());
     if (!special_opset.empty()) {

model-optimizer/automation/package_BOM.txt

@@ -212,6 +212,8 @@ extensions/front/mxnet/instance_norm_ext.py
 extensions/front/mxnet/leaky_relu.py
 extensions/front/mxnet/lrn_ext.py
 extensions/front/mxnet/max_ext.py
+extensions/front/mxnet/modulated_deformable_conv_ext.py
+extensions/front/mxnet/modulated_deformable_conv_replacer.py
 extensions/front/mxnet/multibox_detection_ext.py
 extensions/front/mxnet/mx_reshape_reverse.py
 extensions/front/mxnet/mx_reshape_to_reshape.py

model-optimizer/extensions/back/blob_normalizer.py

@@ -46,7 +46,7 @@ class BlobNormalizer(BackReplacementPattern):
     def find_and_replace_pattern(self, graph: Graph):
         for node in graph.get_op_nodes():
             if node.soft_get('type').lower() not in OpVersioning.opset_1_types and \
-                    not node.soft_get('version') in ["opset2", "opset3", "opset4"]:
+                    not node.soft_get('version') in ["opset2", "opset3", "opset4", "opset8"]:
                 continue
 
             for _, d in node.in_edges().items():

model-optimizer/extensions/back/op_versioning.py

@@ -33,7 +33,6 @@ class OpVersioning(BackReplacementPattern):
         "ConvolutionBackpropData",
         "Cos",
         "Cosh",
-        "DeformableConvolution",
         "DeformablePSROIPooling",
         "DepthToSpace",
         "DetectionOutput",

model-optimizer/extensions/front/mxnet/modulated_deformable_conv_ext.py

@@ -0,0 +1,61 @@
+# Copyright (C) 2018-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import numpy as np
+
+from mo.front.extractor import FrontExtractorOp
+from mo.front.mxnet.extractors.utils import get_mxnet_layer_attrs
+from mo.ops.deformable_convolution import DeformableConvolution
+
+
+class ModulatedDeformableConvolutionExtractor(FrontExtractorOp):
+    op = '_contrib_ModulatedDeformableConvolution'
+    enabled = True
+
+    @classmethod
+    def extract(cls, node):
+        attr = get_mxnet_layer_attrs(node.symbol_dict)
+
+        kernel = attr.tuple("kernel", int, None)
+        stride = attr.tuple("stride", int, tuple(np.ones(len(kernel), dtype=np.int64)))
+        padding = attr.tuple("pad", int, tuple(np.zeros(len(kernel), dtype=np.int64)))
+        dilate = attr.tuple("dilate", int, tuple(np.ones(len(kernel), dtype=np.int64)))
+        num_deformable_group = attr.int("num_deformable_group", 1)
+        num_group = attr.int("num_group", 1)
+        output = attr.int("num_filter", None)
+        bias_term = attr.str("no_bias", 'False') == 'False'
+
+        final_dilations = np.array([1, 1, *[d for d in dilate]], dtype=np.int64) if dilate is not None else None
+
+        node_attrs = {
+            'op': __class__.op,
+            'bias_addable': True,
+            'bias_term': bias_term,
+            'pad': np.array([[0, 0], [0, 0], *[[pad, pad] for pad in padding]], dtype=np.int64),
+            'pad_spatial_shape': np.array([[pad, pad] for pad in padding], dtype=np.int64),
+            'dilation': final_dilations,
+            'output_spatial_shape': None,
+            'output_shape': None,
+            'stride': np.array([1, 1, *[s for s in stride]], dtype=np.int64),
+            'group': num_group,
+            'deformable_group': num_deformable_group,
+            'output': output,
+            'kernel_spatial': np.array([k for k in kernel], dtype=np.int64),
+            'bilinear_interpolation_pad': True,
+
+            'input_feature_channel': 1,
+            'output_feature_channel': 0,
+            'kernel_spatial_idx': None,
+            'reshape_kernel': True,
+            'weights_index': 2,
+            'in_ports_count': 4,
+
+            'spatial_dims': None,
+            'channel_dims': np.array([1], dtype=np.int64),
+            'batch_dims': np.array([0], dtype=np.int64),
+            'layout': 'NCHW',
+        }
+
+        # update the attributes of the node
+        DeformableConvolution.update_node_stat(node, node_attrs)
+        return cls.enabled

model-optimizer/extensions/front/mxnet/modulated_deformable_conv_replacer.py

@@ -0,0 +1,22 @@
+# Copyright (C) 2018-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from mo.front.common.replacement import FrontReplacementPattern
+from mo.graph.graph import Graph
+
+
+class DeformableConvolutionReplacer(FrontReplacementPattern):
+    # swap mask and weights inputs for ModulatedDeformableConvolution according to the specification
+    enabled = True
+
+    def find_and_replace_pattern(self, graph: Graph):
+
+        for deform_conv in graph.get_op_nodes(type='DeformableConvolution'):
+            if len(deform_conv.get_inputs()) != 4:
+                return
+
+            m_source = deform_conv.in_port(2).get_source()
+            deform_conv.in_port(2).disconnect()
+
+            deform_conv.in_port(3).get_connection().set_destination(deform_conv.in_port(2))
+            m_source.connect(deform_conv.in_port(3))

model-optimizer/mo/ops/convolution.py

@@ -102,7 +102,6 @@ class Convolution(Op):
             node['bias_term'] = len(node.in_nodes()) == 3
 
         weights_index = node.weights_index if node.has_valid('weights_index') else 1
-
         # Reshape weights kernel to original shape
         # In case of caffe or MXNet framework, values for weights have no structured shape like OIHW
         # so we have to reshape weights to normal shape
@@ -237,6 +236,8 @@ class Convolution(Op):
         for n in node.out_nodes():
             node.out_node(n).shape = output_shape
 
+        # bin attribute is used for pre-processing, but it will be deleted in BlobNormalizer transformation
+        # and the blobs (weights, biases) will be represented as inputs to the node
         mark_input_bins(node, start_port=1 if node.type != 'DeformableConvolution' else 2)
         assign_dims_to_weights(node.in_node(weights_index), node.kernel_spatial_idx, node.input_feature_channel,
                                node.output_feature_channel, len(kernel_shape))

model-optimizer/mo/ops/deformable_convolution.py

@@ -13,7 +13,7 @@ class DeformableConvolution(Op):
         super().__init__(graph, {
             'type': __class__.op,
             'op': __class__.op,
-            'version': 'opset1',
+            'version': 'opset8',
             'infer': Convolution.infer,
             'group': 1,
             'deformable_group': 1,
@@ -21,8 +21,10 @@ class DeformableConvolution(Op):
             'multiplication_transparent_ports': [(0, 0), (2, 0)],
             'in_ports_count': 3,
             'out_ports_count': 1,
+            'bilinear_interpolation_pad': False,
         }, attrs)
 
     def backend_attrs(self):
-        # the same attributes as in a regular convolution and one additional attribute 'deformable_group' and 'group'
-        return Convolution(self.graph, {}).backend_attrs() + ['deformable_group', 'group']
+        # the same attributes as in a regular convolution and additional attributes 'deformable_group', 'group'
+        # and 'bilinear_interpolation_pad'
+        return Convolution(self.graph, {}).backend_attrs() + ['deformable_group', 'group', 'bilinear_interpolation_pad']

ngraph/core/include/ngraph/op/deformable_convolution.hpp

@@ -171,7 +171,7 @@ namespace ngraph
                 }
 
             private:
-                int64_t m_bilinear_interpolation_pad;
+                bool m_bilinear_interpolation_pad;
             };
         } // namespace v8
     }     // namespace op