DeformableConv2D custom ONNX op (#4816)

ngraph/core/include/ngraph/op/deformable_convolution.hpp

@@ -26,7 +26,7 @@ namespace ngraph
                 ///
                 /// \param arg                Node that produces the input tensor.
                 /// \param deformable_values  Node producing the deformable values tensor.
-                /// \param filters            Node producing the filters(kernels) tensor wit OIZYX
+                /// \param filters            Node producing the filters(kernels) tensor with OIZYX
                 ///                           layout.
                 /// \param strides            Convolution strides.
                 /// \param pads_begin         Amount of padding to be added to the beginning along

ngraph/frontend/onnx_import/src/core/transform.hpp

@@ -42,6 +42,7 @@ namespace ngraph
             void expand_onnx_functions(ONNX_NAMESPACE::ModelProto& model_proto);
 
             static const std::vector<std::string> legacy_ops_to_fixup = {
+                "DeformableConv2D",
                 "DetectionOutput",
                 "ExperimentalDetectronDetectionOutput",
                 "ExperimentalDetectronGenerateProposalsSingleImage",

ngraph/frontend/onnx_import/src/op/org.openvinotoolkit/deformable_conv_2d.cpp

@@ -0,0 +1,51 @@
+//*****************************************************************************
+// Copyright 2017-2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "op/org.openvinotoolkit/deformable_conv_2d.hpp"
+#include "default_opset.hpp"
+#include "ngraph/node.hpp"
+#include "ngraph/op/deformable_convolution.hpp"
+#include "utils/convpool.hpp"
+
+namespace ngraph
+{
+    namespace onnx_import
+    {
+        OutputVector op::set_1::deformable_conv_2d(const Node& node)
+        {
+            const OutputVector& inputs = node.get_ng_inputs();
+            const auto strides = convpool::get_strides(node);
+            const auto dilations = convpool::get_dilations(node);
+            const auto paddings = convpool::get_pads(node);
+
+            const auto group = node.get_attribute_value<int64_t>("group", 1);
+            const auto deformable_groups =
+                node.get_attribute_value<int64_t>("deformable_groups", 1);
+            const auto auto_pad_type = convpool::get_auto_pad(node);
+
+            return {std::make_shared<default_opset::DeformableConvolution>(inputs.at(0),
+                                                                           inputs.at(1),
+                                                                           inputs.at(2),
+                                                                           strides,
+                                                                           paddings.first,
+                                                                           paddings.second,
+                                                                           dilations,
+                                                                           auto_pad_type,
+                                                                           group,
+                                                                           deformable_groups)};
+        }
+    } // namespace onnx_import
+} // namespace ngraph

ngraph/frontend/onnx_import/src/op/org.openvinotoolkit/deformable_conv_2d.hpp

@@ -0,0 +1,38 @@
+//*****************************************************************************
+// Copyright 2017-2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#include "ngraph/node.hpp"
+#include "onnx_import/core/node.hpp"
+
+namespace ngraph
+{
+    namespace onnx_import
+    {
+        namespace op
+        {
+            namespace set_1
+            {
+                OutputVector deformable_conv_2d(const Node& node);
+
+            } // namespace set_1
+
+        } // namespace op
+
+    } // namespace onnx_import
+
+} // namespace ngraph

ngraph/frontend/onnx_import/src/ops_bridge.cpp

@@ -133,6 +133,7 @@
 #include "op/xor.hpp"
 #include "ops_bridge.hpp"
 
+#include "op/org.openvinotoolkit/deformable_conv_2d.hpp"
 #include "op/org.openvinotoolkit/detection_output.hpp"
 #include "op/org.openvinotoolkit/experimental_detectron/detection_output.hpp"
 #include "op/org.openvinotoolkit/experimental_detectron/generate_proposals_single_image.hpp"
@@ -461,6 +462,8 @@ namespace ngraph
             REGISTER_OPERATOR("Xor", 1, logical_xor);
 
             // custom OPs
+            REGISTER_OPERATOR_WITH_DOMAIN(
+                OPENVINO_ONNX_DOMAIN, "DeformableConv2D", 1, deformable_conv_2d);
             REGISTER_OPERATOR_WITH_DOMAIN(
                 OPENVINO_ONNX_DOMAIN, "DetectionOutput", 1, detection_output);
             REGISTER_OPERATOR_WITH_DOMAIN(OPENVINO_ONNX_DOMAIN,

ngraph/test/models/onnx/org.openvinotoolkit/deformable_conv_2d.prototxt

@@ -0,0 +1,115 @@
+ir_version: 7
+producer_name: "nGraph ONNX Importer"
+graph {
+  node {
+    input: "data"
+    input: "deformation"
+    input: "filters"
+    output: "out"
+    op_type: "DeformableConv2D"
+  }
+  name: "test_graph"
+  input {
+    name: "data"
+    type {
+      tensor_type {
+        elem_type: 1
+        shape {
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 4
+          }
+          dim {
+            dim_value: 4
+          }
+        }
+      }
+    }
+  }
+  input {
+    name: "deformation"
+    type {
+      tensor_type {
+        elem_type: 1
+        shape {
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 2
+          }
+          dim {
+            dim_value: 2
+          }
+        }
+      }
+    }
+  }
+  input {
+    name: "filters"
+    type {
+      tensor_type {
+        elem_type: 1
+        shape {
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 2
+          }
+          dim {
+            dim_value: 2
+          }
+        }
+      }
+    }
+  }
+  initializer {
+    name: "filters"
+    dims: 1
+    dims: 1
+    dims: 2
+    dims: 2
+    data_type: 1
+    float_data: 0.1
+    float_data: 0.2
+    float_data: 0.3
+    float_data: 0.4
+  }
+  output {
+    name: "out"
+    type {
+      tensor_type {
+        elem_type: 1
+        shape {
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 3
+          }
+          dim {
+            dim_value: 3
+          }
+        }
+      }
+    }
+  }
+}
+opset_import {
+  version: 7
+}

ngraph/test/onnx/onnx_import_org_openvino.in.cpp

@@ -590,3 +590,45 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_experimental_detectron_topk_rios)
     test_case.add_expected_output<float>(Shape{1, 4}, {1, 1, 3, 4});
     test_case.run();
 }
+
+NGRAPH_TEST(${BACKEND_NAME}, onnx_model_deformable_conv_2d)
+{
+    auto function = onnx_import::import_onnx_model(file_util::path_join(
+        SERIALIZED_ZOO, "onnx/org.openvinotoolkit/deformable_conv_2d.prototxt"));
+
+    auto test_case = test::TestCase<TestEngine>(function);
+
+    // data
+    test_case.add_input<float>({1.0f,
+                                2.0f,
+                                3.0f,
+                                4.0f,
+                                5.0f,
+                                6.0f,
+                                7.0f,
+                                8.0f,
+                                9.0f,
+                                10.0f,
+                                11.0f,
+                                12.0f,
+                                13.0f,
+                                14.0f,
+                                15.0f,
+                                16.0f});
+
+    // deformations
+    test_case.add_input<float>({0.5f, -0.5f, 0.0f, 1.0f});
+
+    test_case.add_expected_output<float>(Shape{1, 1, 3, 3},
+                                         {4.5999999f,
+                                          5.2000003f,
+                                          6.4000001f,
+                                          8.4000006f,
+                                          9.8000002f,
+                                          9.6999998f,
+                                          11.5f,
+                                          13.4000006f,
+                                          14.3999996f});
+
+    test_case.run();
+}

ngraph/test/runtime/ie/unit_test.manifest

@@ -1586,3 +1586,6 @@ bin_convolution_2D_2batch_1channel
 
 # RuntimeError: Unsupported dynamic ops: v4::Interpolate - Ticket: 50691
 onnx_upsample6_dynamic
+
+# random values returned from the plugin: ticket 51762
+onnx_model_deformable_conv_2d

ngraph/test/runtime/interpreter/unit_test.manifest

@@ -112,7 +112,7 @@ INTERPRETER.onnx_model_conv_integer_pads
 onnx_model_lstm_fwd_with_clip_peepholes
 onnx_model_lstm_bdir_short_input_seq_peepholes
 # Activation function hardsigmoid unsupported
-onnx_model_gru_fwd_activations_relu_hardsigmoid 
+onnx_model_gru_fwd_activations_relu_hardsigmoid
 onnx_model_lstm_fwd_hardsigmoid_activation
 gru_cell_hardsigmoid_activation_function
 
@@ -165,3 +165,6 @@ INTERPRETER.onnx_model_experimental_detectron_prior_grid_generator
 
 # Interpreter backend doesn't implement evaluate method for OP ExperimentalDetectronROIFeatureExtractor
 INTERPRETER.onnx_model_experimental_detectron_roi_feature_extractor
+
+# No evaluator for DeformableConv2D
+onnx_model_deformable_conv_2d