From 033274c141be4ee6e119d71ddd07b7080397095e Mon Sep 17 00:00:00 2001 From: Ivan Tikhonov Date: Fri, 2 Jul 2021 19:31:09 +0300 Subject: [PATCH] Deformable convolution v8: ngraph part (#6443) * Deformable convolution: ngraph part * fix docs * add visitor api tests * apply review remarks --- .../convolution/DeformableConvolution_8.md | 13 +- .../ngraph/op/deformable_convolution.hpp | 143 +- .../op/util/deformable_convolution_base.hpp | 84 + .../core/include/ngraph/opsets/opset8_tbl.hpp | 2 +- ngraph/core/src/op/deformable_convolution.cpp | 410 +++-- .../op/util/deformable_convolution_base.cpp | 219 +++ ngraph/test/CMakeLists.txt | 2 + .../deformable_convolution_opset8.cpp | 1424 +++++++++++++++++ .../visitors/op/deformable_convolution.cpp | 76 + 9 files changed, 2130 insertions(+), 243 deletions(-) create mode 100644 ngraph/core/include/ngraph/op/util/deformable_convolution_base.hpp create mode 100644 ngraph/core/src/op/util/deformable_convolution_base.cpp create mode 100644 ngraph/test/type_prop/deformable_convolution_opset8.cpp create mode 100644 ngraph/test/visitors/op/deformable_convolution.cpp diff --git a/docs/ops/convolution/DeformableConvolution_8.md b/docs/ops/convolution/DeformableConvolution_8.md index cf59584a5f4..a9ae67d790d 100644 --- a/docs/ops/convolution/DeformableConvolution_8.md +++ b/docs/ops/convolution/DeformableConvolution_8.md @@ -96,12 +96,12 @@ Where * **Default value**: `1` * **Required**: *no* -* *bilinear_interpolation_padding* +* *bilinear_interpolation_pad* - * **Description**: *bilinear_interpolation_padding* is the number of pixels outside of the feature map boundary to apply bilinear interpolation. - * **Range of values**: non-negative integer value - * **Type**: `int` - * **Default value**: `0` + * **Description**: if *bilinear_interpolation_pad* is `true` and the sampling location is within one pixel outside of the feature map boundary, then bilinear interpolation is performed on the zero padded feature map. If *bilinear_interpolation_pad* is `false` and the sampling location is within one pixel outside of the feature map boundary, then the sampling location shifts to the inner boundary of the feature map. + * **Range of values**: `False` or `True` + * **Type**: `boolean` + * **Default value**: `False` * **Required**: *no* **Inputs**: @@ -112,7 +112,7 @@ Where * **3**: Kernel tensor of type *T* and rank 4. Layout is `OIYX` (number of output channels, number of input channels, spatial axes Y and X). **Required.** -* **4**: ModulationScalars tensor of type *T2* and rank 4, the values are within [0, 1]. Layout is `NCYX` (number of batches, *deformable_group* \* kernel_Y \* kernel_X, spatial axes Y and X). If the input is not provided, the values are assumed to be equal to 1. **Optional.** +* **4**: Mask tensor of type *T* and rank 4. Layout is `NCYX` (number of batches, *deformable_group* \* kernel_Y \* kernel_X, spatial axes Y and X). If the input is not provided, the values are assumed to be equal to 1. **Optional.** **Outputs**: @@ -122,7 +122,6 @@ Where **Types**: * *T*: Any numeric type. -* *T2*: Any supported floating point. **Example** diff --git a/ngraph/core/include/ngraph/op/deformable_convolution.hpp b/ngraph/core/include/ngraph/op/deformable_convolution.hpp index 158a9742049..f55535cb28f 100644 --- a/ngraph/core/include/ngraph/op/deformable_convolution.hpp +++ b/ngraph/core/include/ngraph/op/deformable_convolution.hpp @@ -7,6 +7,7 @@ #include "ngraph/coordinate_diff.hpp" #include "ngraph/op/op.hpp" #include "ngraph/op/util/attr_types.hpp" +#include "ngraph/op/util/deformable_convolution_base.hpp" namespace ngraph { @@ -15,7 +16,7 @@ namespace ngraph namespace v1 { /// \brief DeformableConvolution operation. - class NGRAPH_API DeformableConvolution : public Op + class NGRAPH_API DeformableConvolution : public op::util::DeformableConvolutionBase { public: NGRAPH_RTTI_DECLARATION; @@ -53,39 +54,125 @@ namespace ngraph const PadType& auto_pad = PadType::EXPLICIT, const int64_t group = 1, const int64_t deformable_group = 1); + + std::shared_ptr + clone_with_new_inputs(const OutputVector& new_args) const override; + }; + } // namespace v1 + + namespace v8 + { + class NGRAPH_API DeformableConvolution : public op::util::DeformableConvolutionBase + { + public: + NGRAPH_RTTI_DECLARATION; + + /// \brief Constructs a conversion operation. + DeformableConvolution() = default; + /// \brief Constructs a conversion operation. + /// + /// \param arg Node that produces the input tensor. + /// \param offsets Node producing the deformable values tensor. + /// \param filters Node producing the filters(kernels) tensor with OIZYX + /// layout. + /// \param strides Convolution strides. + /// \param pads_begin Amount of padding to be added to the beginning along + /// each axis. For example in case of a 2D input the value + /// of (1, 2) means that 1 element will be added to the + /// top and 2 elements to the left. + /// \param pads_end Amount of padding to be added to the end along each + /// axis. + /// \param dilations The distance in width and height between the weights + /// in the filters tensor. + /// \param auto_pad Specifies how the automatic calculation of padding + /// should be done. + /// \param group The number of groups which both output and input + /// should be split into. + /// \param deformable_group The number of groups which deformable values and + /// output should be split into along the channel axis. + /// \param bilinear_interpolation_pad + /// The flag that determines the mode of bilinear + /// interpolation execution. + /// If the flag is `true` and the sampling location is + /// within one pixel outside of the feature map boundary, + /// then bilinear interpolation is performed on the zero + /// padded feature map. If the flag is `false` and the + /// sampling location is within one pixel outside of the + /// feature map boundary, then the sampling location + /// shifts to the inner boundary of the feature map.` + DeformableConvolution(const Output& arg, + const Output& offsets, + const Output& filters, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const PadType& auto_pad = PadType::EXPLICIT, + const int64_t group = 1, + const int64_t deformable_group = 1, + const bool bilinear_interpolation_pad = false); + + /// \brief Constructs a conversion operation. + /// + /// \param arg Node that produces the input tensor. + /// \param offsets Node producing the deformable values tensor. + /// \param filters Node producing the filters(kernels) tensor with OIZYX + /// layout. + /// \param mask Node producing the mask(mask) tensor. + /// \param strides Convolution strides. + /// \param pads_begin Amount of padding to be added to the beginning along + /// each axis. For example in case of a 2D input the value + /// of (1, 2) means that 1 element will be added to the + /// top and 2 elements to the left. + /// \param pads_end Amount of padding to be added to the end along each + /// axis. + /// \param dilations The distance in width and height between the weights + /// in the filters tensor. + /// \param auto_pad Specifies how the automatic calculation of padding + /// should be done. + /// \param group The number of groups which both output and input + /// should be split into. + /// \param deformable_group The number of groups which deformable values and + /// output should be split into along the channel axis. + /// \param bilinear_interpolation_pad + /// The flag that determines the mode of bilinear + /// interpolation execution. + /// If the flag is `true` and the sampling location is + /// within one pixel outside of the feature map boundary, + /// then bilinear interpolation is performed on the zero + /// padded feature map. If the flag is `false` and the + /// sampling location is within one pixel outside of the + /// feature map boundary, then the sampling location + /// shifts to the inner boundary of the feature map. + DeformableConvolution(const Output& arg, + const Output& offsets, + const Output& filters, + const Output& mask, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const PadType& auto_pad = PadType::EXPLICIT, + const int64_t group = 1, + const int64_t deformable_group = 1, + const bool bilinear_interpolation_pad = false); bool visit_attributes(AttributeVisitor& visitor) override; void validate_and_infer_types() override; - const Strides& get_strides() const { return m_strides; } - void set_strides(const Strides& strides) { m_strides = strides; } - const Strides& get_dilations() const { return m_dilations; } - void set_dilations(const Strides& dilations) { m_dilations = dilations; } - const CoordinateDiff& get_pads_begin() const { return m_pads_begin; } - void set_pads_begin(const CoordinateDiff& pads_begin) { m_pads_begin = pads_begin; } - const CoordinateDiff& get_pads_end() const { return m_pads_end; } - void set_pads_end(const CoordinateDiff& pads_end) { m_pads_end = pads_end; } - const PadType& get_auto_pad() const { return m_auto_pad; } - void set_auto_pad(const PadType& auto_pad) { m_auto_pad = auto_pad; } - int64_t get_group() const { return m_group; } - void set_group(const int64_t group) { m_group = group; } - int64_t get_deformable_group() const { return m_deformable_group; } - void set_deformable_group(const int64_t deformable_group) - { - m_deformable_group = deformable_group; - } - virtual std::shared_ptr + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - protected: - Strides m_strides; - Strides m_dilations; - CoordinateDiff m_pads_begin; - CoordinateDiff m_pads_end; - PadType m_auto_pad; - int64_t m_group; - int64_t m_deformable_group; + bool get_bilinear_interpolation_pad() const { return m_bilinear_interpolation_pad; } + + void set_bilinear_interpolation_pad(const bool bilinear_interpolation_pad) + { + m_bilinear_interpolation_pad = bilinear_interpolation_pad; + } + + private: + int64_t m_bilinear_interpolation_pad; }; - } // namespace v1 + } // namespace v8 } // namespace op } // namespace ngraph diff --git a/ngraph/core/include/ngraph/op/util/deformable_convolution_base.hpp b/ngraph/core/include/ngraph/op/util/deformable_convolution_base.hpp new file mode 100644 index 00000000000..685f5d631aa --- /dev/null +++ b/ngraph/core/include/ngraph/op/util/deformable_convolution_base.hpp @@ -0,0 +1,84 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "ngraph/coordinate_diff.hpp" +#include "ngraph/op/op.hpp" +#include "ngraph/op/util/attr_types.hpp" + +namespace ngraph +{ + namespace op + { + namespace util + { + /// \brief Base class for operations DeformableConvolution v1 and DeformableConvolution + /// v8. + class NGRAPH_API DeformableConvolutionBase : public Op + { + public: + NGRAPH_RTTI_DECLARATION; + + /// \brief Constructs a conversion operation. + DeformableConvolutionBase() = default; + + /// \brief Constructs a conversion operation. + /// \param strides Convolution strides. + /// \param pads_begin Amount of padding to be added to the beginning along + /// each axis. For example in case of a 2D input the value + /// of (1, 2) means that 1 element will be added to the + /// top and 2 elements to the left. + /// \param pads_end Amount of padding to be added to the end along each + /// axis. + /// \param dilations The distance in width and height between the weights + /// in the filters tensor. + /// \param auto_pad Specifies how the automatic calculation of padding + /// should be done. + /// \param group The number of groups which both output and input + /// should be split into. + /// \param deformable_group The number of groups which deformable values and + /// output should be split into along the channel axis. + DeformableConvolutionBase(const OutputVector& arguments, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const PadType& auto_pad = PadType::EXPLICIT, + int64_t group = 1, + int64_t deformable_group = 1); + + bool visit_attributes(AttributeVisitor& visitor) override; + void validate_and_infer_types() override; + + const Strides& get_strides() const { return m_strides; } + void set_strides(const Strides& strides) { m_strides = strides; } + const Strides& get_dilations() const { return m_dilations; } + void set_dilations(const Strides& dilations) { m_dilations = dilations; } + const CoordinateDiff& get_pads_begin() const { return m_pads_begin; } + void set_pads_begin(const CoordinateDiff& pads_begin) { m_pads_begin = pads_begin; } + const CoordinateDiff& get_pads_end() const { return m_pads_end; } + void set_pads_end(const CoordinateDiff& pads_end) { m_pads_end = pads_end; } + const PadType& get_auto_pad() const { return m_auto_pad; } + void set_auto_pad(const PadType& auto_pad) { m_auto_pad = auto_pad; } + int64_t get_group() const { return m_group; } + void set_group(const int64_t group) { m_group = group; } + int64_t get_deformable_group() const { return m_deformable_group; } + void set_deformable_group(const int64_t deformable_group) + { + m_deformable_group = deformable_group; + } + + protected: + Strides m_strides; + Strides m_dilations; + CoordinateDiff m_pads_begin; + CoordinateDiff m_pads_end; + PadType m_auto_pad; + int64_t m_group; + int64_t m_deformable_group; + }; + } // namespace util + } // namespace op +} // namespace ngraph diff --git a/ngraph/core/include/ngraph/opsets/opset8_tbl.hpp b/ngraph/core/include/ngraph/opsets/opset8_tbl.hpp index 627441eb300..ad4d641027d 100644 --- a/ngraph/core/include/ngraph/opsets/opset8_tbl.hpp +++ b/ngraph/core/include/ngraph/opsets/opset8_tbl.hpp @@ -29,7 +29,6 @@ NGRAPH_OP(ConvolutionBackpropData, ngraph::op::v1) NGRAPH_OP(Cos, ngraph::op::v0) NGRAPH_OP(Cosh, ngraph::op::v0) NGRAPH_OP(CumSum, ngraph::op::v0) -NGRAPH_OP(DeformableConvolution, ngraph::op::v1) NGRAPH_OP(DeformablePSROIPooling, ngraph::op::v1) NGRAPH_OP(DepthToSpace, ngraph::op::v0) NGRAPH_OP(DetectionOutput, ngraph::op::v0) @@ -179,3 +178,4 @@ NGRAPH_OP(Roll, ngraph::op::v7) NGRAPH_OP(Gather, ngraph::op::v8) NGRAPH_OP(AdaptiveAvgPool, ngraph::op::v8) NGRAPH_OP(AdaptiveMaxPool, ngraph::op::v8) +NGRAPH_OP(DeformableConvolution, ngraph::op::v8) diff --git a/ngraph/core/src/op/deformable_convolution.cpp b/ngraph/core/src/op/deformable_convolution.cpp index 7a7278b2449..0c33971ce50 100644 --- a/ngraph/core/src/op/deformable_convolution.cpp +++ b/ngraph/core/src/op/deformable_convolution.cpp @@ -6,14 +6,194 @@ #include "itt.hpp" #include "ngraph/axis_vector.hpp" #include "ngraph/coordinate_diff.hpp" -#include "ngraph/op/reshape.hpp" #include "ngraph/util.hpp" #include "ngraph/validation_util.hpp" using namespace std; using namespace ngraph; -NGRAPH_RTTI_DEFINITION(op::v1::DeformableConvolution, "DeformableConvolution", 1); +NGRAPH_RTTI_DEFINITION(op::v1::DeformableConvolution, + "DeformableConvolution", + 1, + op::util::DeformableConvolutionBase); +NGRAPH_RTTI_DEFINITION(op::v8::DeformableConvolution, + "DeformableConvolution", + 8, + op::util::DeformableConvolutionBase); + +op::v8::DeformableConvolution::DeformableConvolution(const Output& arg, + const Output& offsets, + const Output& filters, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const op::PadType& auto_pad, + const int64_t group, + const int64_t deformable_group, + const bool bilinear_interpolation_pad) + : DeformableConvolutionBase({arg, offsets, filters}, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group) + , m_bilinear_interpolation_pad(bilinear_interpolation_pad) +{ + constructor_validate_and_infer_types(); +} + +op::v8::DeformableConvolution::DeformableConvolution(const Output& arg, + const Output& offsets, + const Output& filters, + const Output& mask, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const op::PadType& auto_pad, + const int64_t group, + const int64_t deformable_group, + const bool bilinear_interpolation_pad) + : DeformableConvolutionBase({arg, offsets, filters, mask}, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group) + , m_bilinear_interpolation_pad(bilinear_interpolation_pad) +{ + constructor_validate_and_infer_types(); +} + +bool op::v8::DeformableConvolution::visit_attributes(AttributeVisitor& visitor) +{ + NGRAPH_OP_SCOPE(DeformableConvolution_v8_visit_attributes); + visitor.on_attribute("bilinear_interpolation_pad", m_bilinear_interpolation_pad); + return DeformableConvolutionBase::visit_attributes(visitor); +} + +void op::v8::DeformableConvolution::validate_and_infer_types() +{ + NGRAPH_OP_SCOPE(DeformableConvolution_v8_validate_and_infer_types); + + DeformableConvolutionBase::validate_and_infer_types(); + if (inputs().size() == 4) + { + const PartialShape& data_pshape = get_input_partial_shape(0); + const PartialShape& filters_pshape = get_input_partial_shape(2); + const PartialShape& mask_pshape = get_input_partial_shape(3); + element::Type mask_et = get_input_element_type(3); + + NODE_VALIDATION_CHECK(this, + mask_et.is_real() || mask_et.is_integral_number(), + "Element type of Mask input must be numeric. Got: ", + mask_et); + + NODE_VALIDATION_CHECK(this, + mask_pshape.rank().compatible(4), + "Mask input must be of rank 4. Got: ", + mask_pshape.rank()); + + if (mask_pshape.rank().is_static() && mask_pshape[1].is_static()) + { + if (filters_pshape.rank().is_static() && filters_pshape[2].is_static() && + filters_pshape[3].is_static()) + { + auto offsets_channels = m_deformable_group * filters_pshape[2].get_length() * + filters_pshape[3].get_length(); + NODE_VALIDATION_CHECK(this, + mask_pshape[1].get_length() == offsets_channels, + "The channels dimension of mask input is not " + "compatible with filters and 'deformable group' attribute. " + "Mask input shape: ", + mask_pshape, + ", deformable 'group' attribute value: ", + m_deformable_group, + ", filters shape: ", + filters_pshape); + } + // At least we can check if mask channels is evenly divisible by deformable + // group attribute + NODE_VALIDATION_CHECK(this, + mask_pshape[1].get_length() % m_deformable_group == 0, + "The channels dimension of mask input must be " + "evenly divisible by the 'deformable group' value along the " + "channels axis. Offsets input shape: ", + mask_pshape, + ", 'deformable group' attribute value: ", + m_deformable_group); + + if (data_pshape.rank().is_static()) + { + NODE_VALIDATION_CHECK( + this, + mask_pshape[0].compatible(data_pshape[0]), + "Data batch and mask batch dimension must be same value. Got: ", + mask_pshape[0], + " and ", + data_pshape[0]); + } + } + + PartialShape result_pshape = get_output_partial_shape(0); + if (result_pshape.rank().is_static() && mask_pshape.rank().is_static()) + { + NODE_VALIDATION_CHECK(this, + result_pshape[2].compatible(mask_pshape[2]) && + result_pshape[3].compatible(mask_pshape[3]), + "Spatial dimensions of mask and output must be equal. Got: ", + mask_pshape[2], + ", ", + mask_pshape[3], + " and ", + result_pshape[2], + ", ", + result_pshape[3]); + } + } +} + +std::shared_ptr + op::v8::DeformableConvolution::clone_with_new_inputs(const OutputVector& new_args) const +{ + NGRAPH_OP_SCOPE(DeformableConvolution_v8_clone_with_new_inputs); + check_new_args_count(this, new_args); + NODE_VALIDATION_CHECK( + this, new_args.size() >= 3 && new_args.size() <= 4, "Number of inputs must be 3 or 4"); + switch (new_args.size()) + { + case 3: + return std::make_shared(new_args.at(0), + new_args.at(1), + new_args.at(2), + m_strides, + m_pads_begin, + m_pads_end, + m_dilations, + m_auto_pad, + m_group, + m_deformable_group, + m_bilinear_interpolation_pad); + default: + return std::make_shared(new_args.at(0), + new_args.at(1), + new_args.at(2), + new_args.at(3), + m_strides, + m_pads_begin, + m_pads_end, + m_dilations, + m_auto_pad, + m_group, + m_deformable_group, + m_bilinear_interpolation_pad); + } +} op::v1::DeformableConvolution::DeformableConvolution(const Output& arg, const Output& offsets, @@ -22,218 +202,34 @@ op::v1::DeformableConvolution::DeformableConvolution(const Output& arg, const CoordinateDiff& pads_begin, const CoordinateDiff& pads_end, const Strides& dilations, - const PadType& auto_pad, + const op::PadType& auto_pad, const int64_t group, const int64_t deformable_group) - : Op({arg, offsets, filters}) - , m_strides(strides) - , m_dilations(dilations) - , m_pads_begin(pads_begin) - , m_pads_end(pads_end) - , m_auto_pad(auto_pad) - , m_group(group) - , m_deformable_group(deformable_group) + : DeformableConvolutionBase({arg, offsets, filters}, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group) { constructor_validate_and_infer_types(); } -bool op::v1::DeformableConvolution::visit_attributes(AttributeVisitor& visitor) -{ - NGRAPH_OP_SCOPE(v1_DeformableConvolution_visit_attributes); - visitor.on_attribute("strides", m_strides); - visitor.on_attribute("dilations", m_dilations); - visitor.on_attribute("pads_begin", m_pads_begin); - visitor.on_attribute("pads_end", m_pads_end); - visitor.on_attribute("auto_pad", m_auto_pad); - visitor.on_attribute("group", m_group); - visitor.on_attribute("deformable_group", m_deformable_group); - return true; -} - -void op::v1::DeformableConvolution::validate_and_infer_types() -{ - NGRAPH_OP_SCOPE(v1_DeformableConvolution_validate_and_infer_types); - const PartialShape& data_batch_pshape = get_input_partial_shape(0); - const PartialShape& offsets_pshape = get_input_partial_shape(1); - const PartialShape& filters_pshape = get_input_partial_shape(2); - - element::Type data_batch_et = get_input_element_type(0); - element::Type offsets_et = get_input_element_type(1); - element::Type filters_et = get_input_element_type(2); - - element::Type result_et; - NODE_VALIDATION_CHECK(this, - element::Type::merge(result_et, data_batch_et, offsets_et) && - element::Type::merge(result_et, result_et, filters_et), - "Element types of inputs do not match. Got: data batch (", - data_batch_et, - "), offsets (", - offsets_et, - ") and filters (", - filters_et, - ")"); - - NODE_VALIDATION_CHECK(this, - result_et.is_real() || result_et.is_integral_number(), - "Element type of inputs must be numeric. Got: ", - result_et); - - Rank result_ps_rank{}; - NODE_VALIDATION_CHECK( - this, - Rank::merge(result_ps_rank, data_batch_pshape.rank(), offsets_pshape.rank()) && - Rank::merge(result_ps_rank, result_ps_rank, filters_pshape.rank()), - "Ranks of inputs do not match. Got: data batch shape ", - data_batch_pshape, - ", offsets shape ", - offsets_pshape, - ", filters shape ", - filters_pshape); - - NODE_VALIDATION_CHECK( - this, result_ps_rank.compatible(4), "Inputs must be of rank 4. Got: ", result_ps_rank); - - NODE_VALIDATION_CHECK( - this, m_group > 0, "Attribute 'group' must be any value starting from 1. Got: ", m_group); - - NODE_VALIDATION_CHECK(this, - m_deformable_group > 0, - "Attribute 'deformable group' must be any value starting from 1. Got: ", - m_deformable_group); - - if (offsets_pshape.rank().is_static()) - { - if (offsets_pshape[1].is_static()) - { - if (filters_pshape.rank().is_static() && filters_pshape[2].is_static() && - filters_pshape[3].is_static()) - { - auto offsets_channels = m_deformable_group * filters_pshape[2].get_length() * - filters_pshape[3].get_length() * 2; - NODE_VALIDATION_CHECK(this, - offsets_pshape[1].get_length() == offsets_channels, - "The channels dimension of offsets input is not " - "compatible with filters and 'deformable group' attribute. " - "Offsets input shape: ", - offsets_pshape, - ", deformable 'group' attribute value: ", - m_deformable_group, - ", filters shape: ", - filters_pshape); - } - else - { - // At least we can check if offsets channels is evenly divisible by deformable - // group attribute - NODE_VALIDATION_CHECK(this, - offsets_pshape[1].get_length() % m_deformable_group == 0, - "The channels dimension of offsets input must be " - "evenly divisible by the 'deformable group' value along the " - "channels axis. Offsets input shape: ", - offsets_pshape, - ", 'deformable group' attribute value: ", - m_deformable_group); - } - } - - if (data_batch_pshape.rank().is_static()) - { - NODE_VALIDATION_CHECK( - this, - offsets_pshape[0].compatible(data_batch_pshape[0]), - "Data batch and offsets batch dimension must be same value. Got: ", - offsets_pshape[0], - " and ", - data_batch_pshape[0]); - } - } - - if (data_batch_pshape.rank().is_static() && data_batch_pshape[1].is_static()) - { - NODE_VALIDATION_CHECK(this, - data_batch_pshape[1].get_length() % m_group == 0, - "The input data shape must be evenly divisible by the 'group' value " - "along the channels axis. Current input shape: ", - data_batch_pshape, - ", 'group' attribute value: ", - m_group); - } - - if (filters_pshape.rank().is_static() && filters_pshape[0].is_static()) - { - NODE_VALIDATION_CHECK( - this, - filters_pshape[0].get_length() % m_group == 0, - "The filters shape must be evenly divisible by the 'group' value along " - "the channels axis. Current filters shape: ", - filters_pshape, - ", 'group' attribute value: ", - m_group); - } - - // adjust filter shape to reuse regular infer_convolution_forward() - const auto new_filters_pshape = [&](int groups) { - auto new_shape(filters_pshape); - if (new_shape.rank().is_static()) - { - new_shape[1] *= groups; - } - return new_shape; - }(m_group); - PartialShape result_shape = - validate_and_infer_convolution_forward_output_shape(this, - result_ps_rank, - data_batch_pshape, - new_filters_pshape, - m_auto_pad, - m_strides, - m_dilations, - m_pads_begin, - m_pads_end); - - if (result_shape.rank().is_static() && offsets_pshape.rank().is_static()) - { - PartialShape result_spatial_shape = [&result_shape]() { - vector result_spatial_dims{result_shape}; - result_spatial_dims.erase(result_spatial_dims.begin(), result_spatial_dims.begin() + 2); - return PartialShape{result_spatial_dims}; - }(); - - PartialShape offsets_spatial_shape = [&offsets_pshape]() { - vector offsets_spatial_dims{offsets_pshape}; - offsets_spatial_dims.erase(offsets_spatial_dims.begin(), - offsets_spatial_dims.begin() + 2); - return PartialShape{offsets_spatial_dims}; - }(); - - NODE_VALIDATION_CHECK(this, - offsets_spatial_shape.compatible(result_spatial_shape), - "Spatial dimensions of offsets and output must be equal. Got: ", - offsets_spatial_shape, - " and ", - result_spatial_shape); - - if (result_shape[0].is_dynamic()) - { - result_shape[0] = offsets_pshape[0]; // batch size - } - } - set_output_type(0, result_et, result_shape); -} - -shared_ptr +std::shared_ptr op::v1::DeformableConvolution::clone_with_new_inputs(const OutputVector& new_args) const { - NGRAPH_OP_SCOPE(v1_DeformableConvolution_clone_with_new_inputs); + NGRAPH_OP_SCOPE(DeformableConvolution_v1_clone_with_new_inputs); check_new_args_count(this, new_args); - return make_shared(new_args.at(0), - new_args.at(1), - new_args.at(2), - m_strides, - m_pads_begin, - m_pads_end, - m_dilations, - m_auto_pad, - m_group, - m_deformable_group); + return std::make_shared(new_args.at(0), + new_args.at(1), + new_args.at(2), + m_strides, + m_pads_begin, + m_pads_end, + m_dilations, + m_auto_pad, + m_group, + m_deformable_group); } diff --git a/ngraph/core/src/op/util/deformable_convolution_base.cpp b/ngraph/core/src/op/util/deformable_convolution_base.cpp new file mode 100644 index 00000000000..f789b7bb22a --- /dev/null +++ b/ngraph/core/src/op/util/deformable_convolution_base.cpp @@ -0,0 +1,219 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "ngraph/op/util/deformable_convolution_base.hpp" +#include "itt.hpp" +#include "ngraph/axis_vector.hpp" +#include "ngraph/coordinate_diff.hpp" +#include "ngraph/op/reshape.hpp" +#include "ngraph/util.hpp" +#include "ngraph/validation_util.hpp" + +using namespace std; +using namespace ngraph; + +NGRAPH_RTTI_DEFINITION(op::util::DeformableConvolutionBase, "DeformableConvolutionBase", 0); + +op::util::DeformableConvolutionBase::DeformableConvolutionBase(const OutputVector& arguments, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const PadType& auto_pad, + const int64_t group, + const int64_t deformable_group) + : Op(arguments) + , m_strides(strides) + , m_dilations(dilations) + , m_pads_begin(pads_begin) + , m_pads_end(pads_end) + , m_auto_pad(auto_pad) + , m_group(group) + , m_deformable_group(deformable_group) +{ +} + +bool op::util::DeformableConvolutionBase::visit_attributes(AttributeVisitor& visitor) +{ + NGRAPH_OP_SCOPE(util_DeformableConvolutionBase_visit_attributes); + visitor.on_attribute("strides", m_strides); + visitor.on_attribute("dilations", m_dilations); + visitor.on_attribute("pads_begin", m_pads_begin); + visitor.on_attribute("pads_end", m_pads_end); + visitor.on_attribute("auto_pad", m_auto_pad); + visitor.on_attribute("group", m_group); + visitor.on_attribute("deformable_group", m_deformable_group); + return true; +} + +void op::util::DeformableConvolutionBase::validate_and_infer_types() +{ + NGRAPH_OP_SCOPE(util_DeformableConvolutionBase_validate_and_infer_types); + const PartialShape& data_batch_pshape = get_input_partial_shape(0); + const PartialShape& offsets_pshape = get_input_partial_shape(1); + const PartialShape& filters_pshape = get_input_partial_shape(2); + + element::Type data_batch_et = get_input_element_type(0); + element::Type offsets_et = get_input_element_type(1); + element::Type filters_et = get_input_element_type(2); + + element::Type result_et; + NODE_VALIDATION_CHECK(this, + element::Type::merge(result_et, data_batch_et, offsets_et) && + element::Type::merge(result_et, result_et, filters_et), + "Element types of inputs do not match. Got: data batch (", + data_batch_et, + "), offsets (", + offsets_et, + ") and filters (", + filters_et, + ")"); + + NODE_VALIDATION_CHECK(this, + result_et.is_real() || result_et.is_integral_number(), + "Element type of inputs must be numeric. Got: ", + result_et); + + Rank result_ps_rank{}; + NODE_VALIDATION_CHECK( + this, + Rank::merge(result_ps_rank, data_batch_pshape.rank(), offsets_pshape.rank()) && + Rank::merge(result_ps_rank, result_ps_rank, filters_pshape.rank()), + "Ranks of inputs do not match. Got: data batch shape ", + data_batch_pshape, + ", offsets shape ", + offsets_pshape, + ", filters shape ", + filters_pshape); + + NODE_VALIDATION_CHECK( + this, result_ps_rank.compatible(4), "Inputs must be of rank 4. Got: ", result_ps_rank); + + NODE_VALIDATION_CHECK( + this, m_group > 0, "Attribute 'group' must be any value starting from 1. Got: ", m_group); + + NODE_VALIDATION_CHECK(this, + m_deformable_group > 0, + "Attribute 'deformable group' must be any value starting from 1. Got: ", + m_deformable_group); + + if (offsets_pshape.rank().is_static()) + { + if (offsets_pshape[1].is_static()) + { + if (filters_pshape.rank().is_static() && filters_pshape[2].is_static() && + filters_pshape[3].is_static()) + { + auto offsets_channels = m_deformable_group * filters_pshape[2].get_length() * + filters_pshape[3].get_length() * 2; + NODE_VALIDATION_CHECK(this, + offsets_pshape[1].get_length() == offsets_channels, + "The channels dimension of offsets input is not " + "compatible with filters and 'deformable group' attribute. " + "Offsets input shape: ", + offsets_pshape, + ", deformable 'group' attribute value: ", + m_deformable_group, + ", filters shape: ", + filters_pshape); + } + else + { + // At least we can check if offsets channels is evenly divisible by deformable + // group attribute + NODE_VALIDATION_CHECK(this, + offsets_pshape[1].get_length() % m_deformable_group == 0, + "The channels dimension of offsets input must be " + "evenly divisible by the 'deformable group' value along the " + "channels axis. Offsets input shape: ", + offsets_pshape, + ", 'deformable group' attribute value: ", + m_deformable_group); + } + } + + if (data_batch_pshape.rank().is_static()) + { + NODE_VALIDATION_CHECK( + this, + offsets_pshape[0].compatible(data_batch_pshape[0]), + "Data batch and offsets batch dimension must be same value. Got: ", + offsets_pshape[0], + " and ", + data_batch_pshape[0]); + } + } + + if (data_batch_pshape.rank().is_static() && data_batch_pshape[1].is_static()) + { + NODE_VALIDATION_CHECK(this, + data_batch_pshape[1].get_length() % m_group == 0, + "The input data shape must be evenly divisible by the 'group' value " + "along the channels axis. Current input shape: ", + data_batch_pshape, + ", 'group' attribute value: ", + m_group); + } + + if (filters_pshape.rank().is_static() && filters_pshape[0].is_static()) + { + NODE_VALIDATION_CHECK( + this, + filters_pshape[0].get_length() % m_group == 0, + "The filters shape must be evenly divisible by the 'group' value along " + "the channels axis. Current filters shape: ", + filters_pshape, + ", 'group' attribute value: ", + m_group); + } + + // adjust filter shape to reuse regular infer_convolution_forward() + const auto new_filters_pshape = [&](int groups) { + auto new_shape(filters_pshape); + if (new_shape.rank().is_static()) + { + new_shape[1] *= groups; + } + return new_shape; + }(m_group); + PartialShape result_shape = + validate_and_infer_convolution_forward_output_shape(this, + result_ps_rank, + data_batch_pshape, + new_filters_pshape, + m_auto_pad, + m_strides, + m_dilations, + m_pads_begin, + m_pads_end); + + if (result_shape.rank().is_static() && offsets_pshape.rank().is_static()) + { + PartialShape result_spatial_shape = [&result_shape]() { + vector result_spatial_dims{result_shape}; + result_spatial_dims.erase(result_spatial_dims.begin(), result_spatial_dims.begin() + 2); + return PartialShape{result_spatial_dims}; + }(); + + PartialShape offsets_spatial_shape = [&offsets_pshape]() { + vector offsets_spatial_dims{offsets_pshape}; + offsets_spatial_dims.erase(offsets_spatial_dims.begin(), + offsets_spatial_dims.begin() + 2); + return PartialShape{offsets_spatial_dims}; + }(); + + NODE_VALIDATION_CHECK(this, + offsets_spatial_shape.compatible(result_spatial_shape), + "Spatial dimensions of offsets and output must be equal. Got: ", + offsets_spatial_shape, + " and ", + result_spatial_shape); + + if (result_shape[0].is_dynamic()) + { + result_shape[0] = offsets_pshape[0]; // batch size + } + } + set_output_type(0, result_et, result_shape); +} diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 5f116c7198d..f516c0d0708 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -115,6 +115,7 @@ set(SRC type_prop/ctc_greedy_decoder_seq_len.cpp type_prop/ctc_loss.cpp type_prop/deformable_convolution.cpp + type_prop/deformable_convolution_opset8.cpp type_prop/deformable_psroi_pooling.cpp type_prop/detection_output.cpp type_prop/depth_to_space.cpp @@ -234,6 +235,7 @@ set(SRC visitors/op/convolution_backprop.cpp visitors/op/cos.cpp visitors/op/cum_sum.cpp + visitors/op/deformable_convolution.cpp visitors/op/deformable_psroi_pooling.cpp visitors/op/depth_to_space.cpp visitors/op/detection_output.cpp diff --git a/ngraph/test/type_prop/deformable_convolution_opset8.cpp b/ngraph/test/type_prop/deformable_convolution_opset8.cpp new file mode 100644 index 00000000000..58e06d06eee --- /dev/null +++ b/ngraph/test/type_prop/deformable_convolution_opset8.cpp @@ -0,0 +1,1424 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "ngraph/opsets/opset8.hpp" +#include "util/type_prop.hpp" + +using namespace std; +using namespace ngraph; +using namespace ngraph::opset8; + +TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 5, 5}; + const PartialShape filters_pshape{4, 1, 3, 3}; + const element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); +} + +TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_lower_data_batch_nc_dims_dynamic) +{ + const PartialShape data_batch_pshape{Dimension::dynamic(), Dimension::dynamic(), 5, 5}; + const PartialShape offsets_pshape{Dimension::dynamic(), 36, 5, 5}; + const PartialShape filters_pshape{4, 4, 3, 3}; + const element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( + PartialShape{Dimension::dynamic(), 4, 5, 5})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); +} + +TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_upper_data_batch_nc_dims_dynamic) +{ + const PartialShape data_batch_pshape{Dimension::dynamic(), Dimension::dynamic(), 5, 5}; + const PartialShape offsets_pshape{1, 16, 5, 5}; + const PartialShape filters_pshape{4, 4, 2, 2}; + const element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_UPPER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); +} + +TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_spatial_dims_dynamic) +{ + const PartialShape data_batch_pshape{1, 4, Dimension::dynamic(), 5}; + const PartialShape offsets_pshape{1, 36, 5, 5}; + const PartialShape filters_pshape{4, 1, 3, 3}; + const element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + + ASSERT_TRUE( + deformable_conv->get_output_partial_shape(0).same_scheme({1, 4, Dimension::dynamic(), 5})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 1})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 1})); +} + +TEST(type_prop, deformable_convolution_opset8_data_batch_dynamic) +{ + const PartialShape data_batch_pshape{PartialShape::dynamic()}; + const PartialShape offsets_pshape{2, 36, 5, 5}; + const PartialShape filters_pshape{4, 4, 3, 3}; + const element::Type_t et = element::f32; + + const auto auto_pad = op::PadType::EXPLICIT; + const int64_t group = 2; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad, + group, + deformable_group); + + ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( + PartialShape{2, 4, Dimension::dynamic(), Dimension::dynamic()})); +} + +TEST(type_prop, deformable_convolution_opset8_offsets_dynamic) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{PartialShape::dynamic()}; + const PartialShape filters_pshape{4, 2, 3, 3}; + const element::Type_t et = element::f32; + + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 2; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad, + group, + deformable_group); + + ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); + ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); +} + +TEST(type_prop, deformable_convolution_opset8_auto_pad_same_filters_dynamic) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 3, 3}; + const PartialShape filters_pshape{PartialShape::dynamic()}; + const element::Type_t et = element::f32; + + const auto auto_pad = op::PadType::SAME_UPPER; + const int64_t group = 2; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad, + group, + deformable_group); + + ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_UPPER); + ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( + PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); +} + +TEST(type_prop, deformable_convolution_opset8_deformable_data_batch_and_filters_dynamic) +{ + const PartialShape data_batch_pshape{PartialShape::dynamic()}; + const PartialShape offsets_pshape{1, 36, 3, 3}; + const PartialShape filters_pshape{PartialShape::dynamic()}; + const element::Type_t et = element::f32; + + const auto auto_pad = op::PadType::EXPLICIT; + const int64_t group = 2; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad, + group, + deformable_group); + + ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( + PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); +} + +TEST(type_prop, deformable_convolution_opset8_deformable_all_inputs_dynamic) +{ + const PartialShape dyn_pshape{PartialShape::dynamic()}; + const element::Type_t et = element::f32; + + const auto auto_pad = op::PadType::EXPLICIT; + const int64_t group = 2; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, dyn_pshape); + auto offsets = make_shared(et, dyn_pshape); + auto filters = make_shared(et, dyn_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad, + group, + deformable_group); + + ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + ASSERT_EQ(deformable_conv->get_strides(), (Strides{})); + ASSERT_EQ(deformable_conv->get_dilations(), (Strides{})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{})); + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape::dynamic())); +} + +TEST(type_prop, deformable_convolution_opset8_invalid_et_inputs) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 4, 3, 3}; + const PartialShape filters_pshape{4, 4, 3, 3}; + + element::Type_t float_et = element::f32; + element::Type_t boolean_et = element::boolean; + + try + { + auto data_batch = make_shared(element::f16, data_batch_pshape); + auto offsets = make_shared(float_et, offsets_pshape); + auto filters = make_shared(float_et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // data batch input must be of same element type as filters and deformable values + FAIL() << "Invalid element type of inputs not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Element types of inputs do not match. Got: data batch (f16), " + "offsets (f32) and filters (f32)"); + } + catch (...) + { + FAIL() << "Element types of inputs validation check failed for unexpected reason."; + } + + try + { + auto data_batch = make_shared(float_et, data_batch_pshape); + auto offsets = make_shared(float_et, offsets_pshape); + auto filters = make_shared(element::f16, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // filters input must be of same element type as data batch and deformable values + FAIL() << "Invalid element type of inputs not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Element types of inputs do not match. Got: " + "data batch (f32), offsets (f32) and filters (f16)"); + } + catch (...) + { + FAIL() << "Element types of inputs validation check failed for unexpected reason."; + } + + try + { + auto data_batch = make_shared(float_et, data_batch_pshape); + auto offsets = make_shared(element::f16, offsets_pshape); + auto filters = make_shared(float_et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // deformable values input must be of same element type as data batch and filters + FAIL() << "Invalid element type of inputs not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Element types of inputs do not match. Got: data batch (f32), " + "offsets (f16) and filters (f32)"); + } + catch (...) + { + FAIL() << "Element types of inputs validation check failed for unexpected reason."; + } + + try + { + auto data_batch = make_shared(boolean_et, data_batch_pshape); + auto offsets = make_shared(boolean_et, offsets_pshape); + auto filters = make_shared(boolean_et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // element types for inputs must be numeric + FAIL() << "Invalid boolean element type of inputs not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Element type of inputs must be numeric"); + } + catch (...) + { + FAIL() << "Numeric element types of inputs validation check failed for " + "unexpected reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_input_ranks) +{ + const element::Type_t et = element::f32; + + // data batch shape provides is rank 5 + try + { + const PartialShape data_batch_pshape{1, 4, 5, 5, 5}; + const PartialShape offsets_pshape{1, 4, 4, 4}; + const PartialShape filters_pshape{4, 4, 3, 3}; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // data batch has invalid rank 5, should be 4 + FAIL() << "Incompatible data batch input rank not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Ranks of inputs do not match. Got: data batch " + "shape {1,4,5,5,5}, offsets shape {1,4,4,4}, filters " + "shape {4,4,3,3}"); + } + catch (...) + { + FAIL() << "Rank validation check of data batch input failed for unexpected reason"; + } + + // deformable values shape provides is rank 5 + try + { + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 4, 4, 4, 4}; + const PartialShape filters_pshape{4, 4, 3, 3}; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // deformable values has invalid rank 5, should be 4 + FAIL() << "Incompatible offsets input rank not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Ranks of inputs do not match. Got: data batch shape " + "{1,4,5,5}, offsets shape {1,4,4,4,4}, filters shape " + "{4,4,3,3}"); + } + catch (...) + { + FAIL() << "Rank validation check of offsets input failed for unexpected reason"; + } + + // filter partial shape provided is rank 5 + try + { + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 4, 4, 4}; + const PartialShape filters_pshape{4, 4, 3, 3, 3}; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // filters has invalid rank 5, should be 4 + FAIL() << "Incompatible filter input rank not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Ranks of inputs do not match. Got: data batch shape " + "{1,4,5,5}, offsets shape {1,4,4,4}, filters shape " + "{4,4,3,3,3}"); + } + catch (...) + { + FAIL() << "Rank validation check of filter input failed for unexpected reason"; + } + + // 3D deformable convolution (not supported) + try + { + const PartialShape data_batch_pshape{1, 4, 5, 5, 5}; + const PartialShape offsets_pshape{1, 4, 4, 4, 4}; + const PartialShape filters_pshape{4, 4, 3, 3, 3}; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // inputs have rank 5, should be 4 + FAIL() << "Incompatible input ranks not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Inputs must be of rank 4"); + } + catch (...) + { + FAIL() << "Rank validation check for 2 spatial dimension inputs failed for unexpected reason"; + } + + // 1D deformable convolution (not supported) + try + { + const PartialShape data_batch_pshape{1, 4, 5}; + const PartialShape offsets_pshape{1, 4, 4}; + const PartialShape filters_pshape{4, 4, 3}; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}); + // inputs have rank 3, should be 4 + FAIL() << "Incompatible input ranks not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Inputs must be of rank 4"); + } + catch (...) + { + FAIL() << "Rank validation check for 2 spatial dimension inputs failed for unexpected reason"; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_groups) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 4, 3, 3}; + const PartialShape filters_pshape{4, 4, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 0; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // attribute group is invalid + FAIL() << "Invalid attribute group value not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Attribute 'group' must be any value starting from 1"); + } + catch (...) + { + FAIL() << "Attribute group validation check failed for unexpected " + "reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_deformable_groups) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 4, 3, 3}; + const PartialShape filters_pshape{4, 4, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 0; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // attribute deformable group is invalid + FAIL() << "Invalid attribute deformable group value not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Attribute 'deformable group' must be any value starting from 1"); + } + catch (...) + { + FAIL() << "Attribute deformable group validation check failed for unexpected " + "reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_offsets_channels_dim) +{ + try + { + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 9, 3, 3}; + const PartialShape filters_pshape{4, 4, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // Channels dim of deformable values is incorrect. Should be 36 + FAIL() << "Invalid channels dimension of offsets input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The channels dimension of offsets input is not " + "compatible with filters and 'deformable group' attribute"); + } + catch (...) + { + FAIL() << "Channels dimension of offsets input validation check failed for " + "unexpected " + "reason."; + } + + // filters spatial dims are dynamic + // we can still check if channels dim of offsets if evenly + // divisible by deformable group attribute + try + { + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 35, Dimension::dynamic(), Dimension::dynamic()}; + const PartialShape filters_pshape{4, 4, Dimension::dynamic(), Dimension::dynamic()}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // Channels dim of deformable values is incorrect + FAIL() << "Invalid channels dimension of offsets input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The channels dimension of offsets input must be " + "evenly divisible by the 'deformable group' value along the " + "channels axis."); + } + catch (...) + { + FAIL() << "Channels dimension of offsets input validation check failed for " + "unexpected reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_offsets_batch_dim) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{2, 36, 3, 3}; + const PartialShape filters_pshape{4, 4, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // data batch and deformable values inputs must have the same batch dimension + FAIL() << "Invalid batch dimension of offsets input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Data batch and offsets batch dimension must be same value"); + } + catch (...) + { + FAIL() + << "Batch dimension of offsets input validation check failed for unexpected " + "reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_data_batch_channels_dim_with_group) +{ + const PartialShape data_batch_pshape{1, 5, 5, 5}; + const PartialShape offsets_pshape{1, 36, 3, 3}; + const PartialShape filters_pshape{4, 5, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::EXPLICIT; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // data batch channels is not evenly divisible by the attribute group value + FAIL() << "Invalid channels dimension of data batch input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The input data shape must be evenly divisible by the 'group' value " + "along the channels axis."); + } + catch (...) + { + FAIL() << "Data batch channel dimension validation check failed for unexpected " + "reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_filters_channels_dim_with_group) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 3, 3}; + const PartialShape filters_pshape{5, 4, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::EXPLICIT; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // filters channels output is not evenly divisible by the attribute group value + FAIL() << "Invalid channels output dimension of filters input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING( + error.what(), + "The filters shape must be evenly divisible by the 'group' value along " + "the channels axis"); + } + catch (...) + { + FAIL() << "Filters channels output dimension validation check failed for unexpected " + "reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_incompatible_data_batch_and_filters_channels_dim) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 3, 3}; + const PartialShape filters_pshape{4, 4, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::EXPLICIT; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // data batch and filters should have same channels input dimension + FAIL() << "Incompatible channels dimension of data batch and filters input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING( + error.what(), + "Data batch channel count (4) does not match filter input channel count (16)"); + } + catch (...) + { + FAIL() << "Data batch channel and filter channel dimension validation check failed for " + "unexpected " + "reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_offsets_spatial_dims) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 4, 4}; + const PartialShape filters_pshape{4, 1, 3, 3}; + const element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // deformable values has incorrect spatial dimensions + FAIL() << "Invalid spatial dimensions of offsets not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Spatial dimensions of offsets and output must be equal"); + } + catch (...) + { + FAIL() << "Spatial dimension of offsets validation check failed for unexpected reason"; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_conv_param_spatial_dims) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 18, 3, 3}; + const PartialShape filters_pshape{4, 4, 3, 3}; + element::Type_t et = element::f32; + + // invalid strides spatial dimensions + try + { + Strides strides{1, 1, 1}; + Strides dilations{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, PartialShape::dynamic()); + auto deformable_conv = make_shared( + data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); + FAIL() << "Invalid strides spatial dimensions not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Strides should be defined for all and only spatial features."); + } + catch (...) + { + FAIL() << "Strides spatial dimensions validation check failed for unexpected reason"; + } + try + { + Strides strides{1}; + Strides dilations{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + + auto data_batch = make_shared(et, PartialShape::dynamic()); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared( + data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); + FAIL() << "Invalid strides spatial dimensions not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Strides should be defined for all and only spatial features."); + } + catch (...) + { + FAIL() << "Strides spatial dimensions validation check failed for unexpected reason"; + } + + // invalid dilations spatial dimensions + try + { + Strides strides{1, 1}; + Strides dilations{1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, PartialShape::dynamic()); + auto deformable_conv = make_shared( + data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); + FAIL() << "Invalid dilations spatial dimensions not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Dilations should be defined for all and only spatial features."); + } + catch (...) + { + FAIL() << "Dilations spatial dimensions validation check failed for unexpected reason"; + } + try + { + Strides strides{1, 1}; + Strides dilations{1, 1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + + auto data_batch = make_shared(et, PartialShape::dynamic()); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared( + data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); + FAIL() << "Invalid dilations spatial dimensions not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Dilations should be defined for all and only spatial features."); + } + catch (...) + { + FAIL() << "Dilations spatial dimensions validation check failed for unexpected reason"; + } + + // invalid padding spatial dimensions + try + { + Strides strides{1, 1}; + Strides dilations{1, 1}; + CoordinateDiff pads_begin{0, 0, 0}; + CoordinateDiff pads_end{0, 0}; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, PartialShape::dynamic()); + auto deformable_conv = make_shared( + data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); + FAIL() << "Invalid padding spatial dimensions not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Pads should be defined for all and only spatial features."); + } + catch (...) + { + FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; + } + try + { + Strides strides{1, 1}; + Strides dilations{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0}; + + auto data_batch = make_shared(et, PartialShape::dynamic()); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto deformable_conv = make_shared( + data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); + FAIL() << "Invalid padding spatial dimensions not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Pads should be defined for all and only spatial features."); + } + catch (...) + { + FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_mask_spatial_dims) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 5, 5}; + const PartialShape mask_pshape{1, 18, 4, 4}; + const PartialShape filters_pshape{4, 1, 3, 3}; + const element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto mask = make_shared(et, mask_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + mask, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // deformable values has incorrect spatial dimensions + FAIL() << "Invalid spatial dimensions of mask not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Spatial dimensions of mask and output must be equal"); + } + catch (...) + { + FAIL() << "Spatial dimension of mask validation check failed for unexpected reason"; + } +} + +TEST(type_prop, deformable_convolution_opset8_mask_dynamic) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 5, 5}; + const PartialShape filters_pshape{4, 2, 3, 3}; + const PartialShape mask_pshape{PartialShape::dynamic()}; + const element::Type_t et = element::f32; + + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 2; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto mask = make_shared(et, mask_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + mask, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad, + group, + deformable_group); + + ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); + ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); +} + +TEST(type_prop, deformable_convolution_opset8_invalid_mask_channels_dim) +{ + try + { + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 5, 5}; + const PartialShape filters_pshape{4, 1, 3, 3}; + const PartialShape mask_pshape{1, 9, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto mask = make_shared(et, mask_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + mask, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // Channels dim of deformable values is incorrect. Should be 18 + FAIL() << "Invalid channels dimension of mask input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The channels dimension of mask input is not " + "compatible with filters and 'deformable group' attribute"); + } + catch (...) + { + FAIL() << "Channels dimension of mask input validation check failed for " + "unexpected " + "reason."; + } + + // mask spatial dims are dynamic + // we can still check if channels dim of offsets if evenly + // divisible by deformable group attribute + try + { + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, Dimension::dynamic(), Dimension::dynamic()}; + const PartialShape filters_pshape{4, 4, Dimension::dynamic(), Dimension::dynamic()}; + const PartialShape mask_pshape{4, 9, Dimension::dynamic(), Dimension::dynamic()}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto mask = make_shared(et, mask_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + mask, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // Channels dim of deformable values is incorrect + FAIL() << "Invalid channels dimension of mask input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The channels dimension of mask input must be " + "evenly divisible by the 'deformable group' value along the " + "channels axis."); + } + catch (...) + { + FAIL() << "Channels dimension of mask input validation check failed for " + "unexpected reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_invalid_mask_batch_dim) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 5, 5}; + const PartialShape filters_pshape{4, 1, 3, 3}; + const PartialShape mask_pshape{2, 18, 3, 3}; + element::Type_t et = element::f32; + + Strides strides{1, 1}; + CoordinateDiff pads_begin{0, 0}; + CoordinateDiff pads_end{0, 0}; + Strides dilations{1, 1}; + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 4; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto mask = make_shared(et, mask_pshape); + + try + { + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + mask, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + group, + deformable_group); + // data batch and deformable values inputs must have the same batch dimension + FAIL() << "Invalid batch dimension of mask input not detected"; + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Data batch and mask batch dimension must be same value"); + } + catch (...) + { + FAIL() + << "Batch dimension of mask input validation check failed for unexpected " + "reason."; + } +} + +TEST(type_prop, deformable_convolution_opset8_mask) +{ + const PartialShape data_batch_pshape{1, 4, 5, 5}; + const PartialShape offsets_pshape{1, 36, 5, 5}; + const PartialShape filters_pshape{4, 2, 3, 3}; + const PartialShape mask_pshape{1, 18, 5, 5}; + const element::Type_t et = element::f32; + + const auto auto_pad = op::PadType::SAME_LOWER; + const int64_t group = 2; + const int64_t deformable_group = 2; + + auto data_batch = make_shared(et, data_batch_pshape); + auto offsets = make_shared(et, offsets_pshape); + auto filters = make_shared(et, filters_pshape); + auto mask = make_shared(et, mask_pshape); + auto deformable_conv = make_shared(data_batch, + offsets, + filters, + mask, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad, + group, + deformable_group); + + ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); + ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); +} diff --git a/ngraph/test/visitors/op/deformable_convolution.cpp b/ngraph/test/visitors/op/deformable_convolution.cpp new file mode 100644 index 00000000000..724a7ec0c06 --- /dev/null +++ b/ngraph/test/visitors/op/deformable_convolution.cpp @@ -0,0 +1,76 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" + +#include "ngraph/ngraph.hpp" +#include "ngraph/op/util/attr_types.hpp" +#include "ngraph/opsets/opset8.hpp" + +#include "util/visitor.hpp" + +using namespace std; +using namespace ngraph; +using ngraph::test::NodeBuilder; +using ngraph::test::ValueMap; + +TEST(attributes, deformable_convolution_default_attributes) +{ + NodeBuilder::get_ops().register_factory(); + const Shape inputs_shape{1, 1, 5, 5}; + auto data = make_shared(element::f32, Shape{1, 1, 5, 5}); + auto filters = make_shared(element::f32, Shape{1, 1, 3, 3}); + auto offsets = make_shared(element::f32, Shape{1, 18, 3, 3}); + auto strides = Strides{1, 1}; + auto pads_begin = CoordinateDiff{0, 0}; + auto pads_end = CoordinateDiff{0, 0}; + auto dilations = Strides{1, 1}; + auto convolution = make_shared(data, offsets, filters, strides, pads_begin, pads_end, dilations); + NodeBuilder builder(convolution); + auto g_convolution = as_type_ptr(builder.create()); + + // attribute count + const auto expected_attr_count = 8; + EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); + + EXPECT_EQ(g_convolution->get_strides(), convolution->get_strides()); + EXPECT_EQ(g_convolution->get_pads_begin(), convolution->get_pads_begin()); + EXPECT_EQ(g_convolution->get_pads_end(), convolution->get_pads_end()); + EXPECT_EQ(g_convolution->get_dilations(), convolution->get_dilations()); + EXPECT_EQ(g_convolution->get_auto_pad(), convolution->get_auto_pad()); + EXPECT_EQ(g_convolution->get_group(), convolution->get_group()); + EXPECT_EQ(g_convolution->get_deformable_group(), convolution->get_deformable_group()); + EXPECT_EQ(g_convolution->get_bilinear_interpolation_pad(), convolution->get_bilinear_interpolation_pad()); +} + +TEST(attributes, deformable_convolution_attributes) +{ + NodeBuilder::get_ops().register_factory(); + const Shape inputs_shape{1, 1, 5, 5}; + auto data = make_shared(element::f32, Shape{1, 2, 5, 5}); + auto filters = make_shared(element::f32, Shape{2, 1, 3, 3}); + auto offsets = make_shared(element::f32, Shape{1, 36, 5, 5}); + auto mask = make_shared(element::f32, Shape{1, 18, 5, 5}); + auto strides = Strides{1, 1}; + auto pads_begin = CoordinateDiff{0, 0}; + auto pads_end = CoordinateDiff{0, 0}; + auto dilations = Strides{1, 1}; + auto convolution = make_shared(data, offsets, filters, mask, strides, pads_begin, pads_end, dilations, + op::PadType::SAME_LOWER, 2, 2, true); + NodeBuilder builder(convolution); + auto g_convolution = as_type_ptr(builder.create()); + + // attribute count + const auto expected_attr_count = 8; + EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); + + EXPECT_EQ(g_convolution->get_strides(), convolution->get_strides()); + EXPECT_EQ(g_convolution->get_pads_begin(), convolution->get_pads_begin()); + EXPECT_EQ(g_convolution->get_pads_end(), convolution->get_pads_end()); + EXPECT_EQ(g_convolution->get_dilations(), convolution->get_dilations()); + EXPECT_EQ(g_convolution->get_auto_pad(), convolution->get_auto_pad()); + EXPECT_EQ(g_convolution->get_group(), convolution->get_group()); + EXPECT_EQ(g_convolution->get_deformable_group(), convolution->get_deformable_group()); + EXPECT_EQ(g_convolution->get_bilinear_interpolation_pad(), convolution->get_bilinear_interpolation_pad()); +}