Deformable convolution v8: ngraph part (#6443)

* Deformable convolution: ngraph part

* fix docs

* add visitor api tests

* apply review remarks
This commit is contained in:
Ivan Tikhonov 2021-07-02 19:31:09 +03:00 committed by GitHub
parent 9d9bba62a4
commit 033274c141
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 2130 additions and 243 deletions

View File

@ -96,12 +96,12 @@ Where
* **Default value**: `1`
* **Required**: *no*
* *bilinear_interpolation_padding*
* *bilinear_interpolation_pad*
* **Description**: *bilinear_interpolation_padding* is the number of pixels outside of the feature map boundary to apply bilinear interpolation.
* **Range of values**: non-negative integer value
* **Type**: `int`
* **Default value**: `0`
* **Description**: if *bilinear_interpolation_pad* is `true` and the sampling location is within one pixel outside of the feature map boundary, then bilinear interpolation is performed on the zero padded feature map. If *bilinear_interpolation_pad* is `false` and the sampling location is within one pixel outside of the feature map boundary, then the sampling location shifts to the inner boundary of the feature map.
* **Range of values**: `False` or `True`
* **Type**: `boolean`
* **Default value**: `False`
* **Required**: *no*
**Inputs**:
@ -112,7 +112,7 @@ Where
* **3**: Kernel tensor of type *T* and rank 4. Layout is `OIYX` (number of output channels, number of input channels, spatial axes Y and X). **Required.**
* **4**: ModulationScalars tensor of type *T2* and rank 4, the values are within [0, 1]. Layout is `NCYX` (number of batches, *deformable_group* \* kernel_Y \* kernel_X, spatial axes Y and X). If the input is not provided, the values are assumed to be equal to 1. **Optional.**
* **4**: Mask tensor of type *T* and rank 4. Layout is `NCYX` (number of batches, *deformable_group* \* kernel_Y \* kernel_X, spatial axes Y and X). If the input is not provided, the values are assumed to be equal to 1. **Optional.**
**Outputs**:
@ -122,7 +122,6 @@ Where
**Types**:
* *T*: Any numeric type.
* *T2*: Any supported floating point.
**Example**

View File

@ -7,6 +7,7 @@
#include "ngraph/coordinate_diff.hpp"
#include "ngraph/op/op.hpp"
#include "ngraph/op/util/attr_types.hpp"
#include "ngraph/op/util/deformable_convolution_base.hpp"
namespace ngraph
{
@ -15,7 +16,7 @@ namespace ngraph
namespace v1
{
/// \brief DeformableConvolution operation.
class NGRAPH_API DeformableConvolution : public Op
class NGRAPH_API DeformableConvolution : public op::util::DeformableConvolutionBase
{
public:
NGRAPH_RTTI_DECLARATION;
@ -53,39 +54,125 @@ namespace ngraph
const PadType& auto_pad = PadType::EXPLICIT,
const int64_t group = 1,
const int64_t deformable_group = 1);
std::shared_ptr<Node>
clone_with_new_inputs(const OutputVector& new_args) const override;
};
} // namespace v1
namespace v8
{
class NGRAPH_API DeformableConvolution : public op::util::DeformableConvolutionBase
{
public:
NGRAPH_RTTI_DECLARATION;
/// \brief Constructs a conversion operation.
DeformableConvolution() = default;
/// \brief Constructs a conversion operation.
///
/// \param arg Node that produces the input tensor.
/// \param offsets Node producing the deformable values tensor.
/// \param filters Node producing the filters(kernels) tensor with OIZYX
/// layout.
/// \param strides Convolution strides.
/// \param pads_begin Amount of padding to be added to the beginning along
/// each axis. For example in case of a 2D input the value
/// of (1, 2) means that 1 element will be added to the
/// top and 2 elements to the left.
/// \param pads_end Amount of padding to be added to the end along each
/// axis.
/// \param dilations The distance in width and height between the weights
/// in the filters tensor.
/// \param auto_pad Specifies how the automatic calculation of padding
/// should be done.
/// \param group The number of groups which both output and input
/// should be split into.
/// \param deformable_group The number of groups which deformable values and
/// output should be split into along the channel axis.
/// \param bilinear_interpolation_pad
/// The flag that determines the mode of bilinear
/// interpolation execution.
/// If the flag is `true` and the sampling location is
/// within one pixel outside of the feature map boundary,
/// then bilinear interpolation is performed on the zero
/// padded feature map. If the flag is `false` and the
/// sampling location is within one pixel outside of the
/// feature map boundary, then the sampling location
/// shifts to the inner boundary of the feature map.`
DeformableConvolution(const Output<Node>& arg,
const Output<Node>& offsets,
const Output<Node>& filters,
const Strides& strides,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const Strides& dilations,
const PadType& auto_pad = PadType::EXPLICIT,
const int64_t group = 1,
const int64_t deformable_group = 1,
const bool bilinear_interpolation_pad = false);
/// \brief Constructs a conversion operation.
///
/// \param arg Node that produces the input tensor.
/// \param offsets Node producing the deformable values tensor.
/// \param filters Node producing the filters(kernels) tensor with OIZYX
/// layout.
/// \param mask Node producing the mask(mask) tensor.
/// \param strides Convolution strides.
/// \param pads_begin Amount of padding to be added to the beginning along
/// each axis. For example in case of a 2D input the value
/// of (1, 2) means that 1 element will be added to the
/// top and 2 elements to the left.
/// \param pads_end Amount of padding to be added to the end along each
/// axis.
/// \param dilations The distance in width and height between the weights
/// in the filters tensor.
/// \param auto_pad Specifies how the automatic calculation of padding
/// should be done.
/// \param group The number of groups which both output and input
/// should be split into.
/// \param deformable_group The number of groups which deformable values and
/// output should be split into along the channel axis.
/// \param bilinear_interpolation_pad
/// The flag that determines the mode of bilinear
/// interpolation execution.
/// If the flag is `true` and the sampling location is
/// within one pixel outside of the feature map boundary,
/// then bilinear interpolation is performed on the zero
/// padded feature map. If the flag is `false` and the
/// sampling location is within one pixel outside of the
/// feature map boundary, then the sampling location
/// shifts to the inner boundary of the feature map.
DeformableConvolution(const Output<Node>& arg,
const Output<Node>& offsets,
const Output<Node>& filters,
const Output<Node>& mask,
const Strides& strides,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const Strides& dilations,
const PadType& auto_pad = PadType::EXPLICIT,
const int64_t group = 1,
const int64_t deformable_group = 1,
const bool bilinear_interpolation_pad = false);
bool visit_attributes(AttributeVisitor& visitor) override;
void validate_and_infer_types() override;
const Strides& get_strides() const { return m_strides; }
void set_strides(const Strides& strides) { m_strides = strides; }
const Strides& get_dilations() const { return m_dilations; }
void set_dilations(const Strides& dilations) { m_dilations = dilations; }
const CoordinateDiff& get_pads_begin() const { return m_pads_begin; }
void set_pads_begin(const CoordinateDiff& pads_begin) { m_pads_begin = pads_begin; }
const CoordinateDiff& get_pads_end() const { return m_pads_end; }
void set_pads_end(const CoordinateDiff& pads_end) { m_pads_end = pads_end; }
const PadType& get_auto_pad() const { return m_auto_pad; }
void set_auto_pad(const PadType& auto_pad) { m_auto_pad = auto_pad; }
int64_t get_group() const { return m_group; }
void set_group(const int64_t group) { m_group = group; }
int64_t get_deformable_group() const { return m_deformable_group; }
void set_deformable_group(const int64_t deformable_group)
{
m_deformable_group = deformable_group;
}
virtual std::shared_ptr<Node>
std::shared_ptr<Node>
clone_with_new_inputs(const OutputVector& new_args) const override;
protected:
Strides m_strides;
Strides m_dilations;
CoordinateDiff m_pads_begin;
CoordinateDiff m_pads_end;
PadType m_auto_pad;
int64_t m_group;
int64_t m_deformable_group;
bool get_bilinear_interpolation_pad() const { return m_bilinear_interpolation_pad; }
void set_bilinear_interpolation_pad(const bool bilinear_interpolation_pad)
{
m_bilinear_interpolation_pad = bilinear_interpolation_pad;
}
private:
int64_t m_bilinear_interpolation_pad;
};
} // namespace v1
} // namespace v8
} // namespace op
} // namespace ngraph

View File

@ -0,0 +1,84 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "ngraph/coordinate_diff.hpp"
#include "ngraph/op/op.hpp"
#include "ngraph/op/util/attr_types.hpp"
namespace ngraph
{
namespace op
{
namespace util
{
/// \brief Base class for operations DeformableConvolution v1 and DeformableConvolution
/// v8.
class NGRAPH_API DeformableConvolutionBase : public Op
{
public:
NGRAPH_RTTI_DECLARATION;
/// \brief Constructs a conversion operation.
DeformableConvolutionBase() = default;
/// \brief Constructs a conversion operation.
/// \param strides Convolution strides.
/// \param pads_begin Amount of padding to be added to the beginning along
/// each axis. For example in case of a 2D input the value
/// of (1, 2) means that 1 element will be added to the
/// top and 2 elements to the left.
/// \param pads_end Amount of padding to be added to the end along each
/// axis.
/// \param dilations The distance in width and height between the weights
/// in the filters tensor.
/// \param auto_pad Specifies how the automatic calculation of padding
/// should be done.
/// \param group The number of groups which both output and input
/// should be split into.
/// \param deformable_group The number of groups which deformable values and
/// output should be split into along the channel axis.
DeformableConvolutionBase(const OutputVector& arguments,
const Strides& strides,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const Strides& dilations,
const PadType& auto_pad = PadType::EXPLICIT,
int64_t group = 1,
int64_t deformable_group = 1);
bool visit_attributes(AttributeVisitor& visitor) override;
void validate_and_infer_types() override;
const Strides& get_strides() const { return m_strides; }
void set_strides(const Strides& strides) { m_strides = strides; }
const Strides& get_dilations() const { return m_dilations; }
void set_dilations(const Strides& dilations) { m_dilations = dilations; }
const CoordinateDiff& get_pads_begin() const { return m_pads_begin; }
void set_pads_begin(const CoordinateDiff& pads_begin) { m_pads_begin = pads_begin; }
const CoordinateDiff& get_pads_end() const { return m_pads_end; }
void set_pads_end(const CoordinateDiff& pads_end) { m_pads_end = pads_end; }
const PadType& get_auto_pad() const { return m_auto_pad; }
void set_auto_pad(const PadType& auto_pad) { m_auto_pad = auto_pad; }
int64_t get_group() const { return m_group; }
void set_group(const int64_t group) { m_group = group; }
int64_t get_deformable_group() const { return m_deformable_group; }
void set_deformable_group(const int64_t deformable_group)
{
m_deformable_group = deformable_group;
}
protected:
Strides m_strides;
Strides m_dilations;
CoordinateDiff m_pads_begin;
CoordinateDiff m_pads_end;
PadType m_auto_pad;
int64_t m_group;
int64_t m_deformable_group;
};
} // namespace util
} // namespace op
} // namespace ngraph

View File

@ -29,7 +29,6 @@ NGRAPH_OP(ConvolutionBackpropData, ngraph::op::v1)
NGRAPH_OP(Cos, ngraph::op::v0)
NGRAPH_OP(Cosh, ngraph::op::v0)
NGRAPH_OP(CumSum, ngraph::op::v0)
NGRAPH_OP(DeformableConvolution, ngraph::op::v1)
NGRAPH_OP(DeformablePSROIPooling, ngraph::op::v1)
NGRAPH_OP(DepthToSpace, ngraph::op::v0)
NGRAPH_OP(DetectionOutput, ngraph::op::v0)
@ -179,3 +178,4 @@ NGRAPH_OP(Roll, ngraph::op::v7)
NGRAPH_OP(Gather, ngraph::op::v8)
NGRAPH_OP(AdaptiveAvgPool, ngraph::op::v8)
NGRAPH_OP(AdaptiveMaxPool, ngraph::op::v8)
NGRAPH_OP(DeformableConvolution, ngraph::op::v8)

View File

@ -6,14 +6,194 @@
#include "itt.hpp"
#include "ngraph/axis_vector.hpp"
#include "ngraph/coordinate_diff.hpp"
#include "ngraph/op/reshape.hpp"
#include "ngraph/util.hpp"
#include "ngraph/validation_util.hpp"
using namespace std;
using namespace ngraph;
NGRAPH_RTTI_DEFINITION(op::v1::DeformableConvolution, "DeformableConvolution", 1);
NGRAPH_RTTI_DEFINITION(op::v1::DeformableConvolution,
"DeformableConvolution",
1,
op::util::DeformableConvolutionBase);
NGRAPH_RTTI_DEFINITION(op::v8::DeformableConvolution,
"DeformableConvolution",
8,
op::util::DeformableConvolutionBase);
op::v8::DeformableConvolution::DeformableConvolution(const Output<Node>& arg,
const Output<Node>& offsets,
const Output<Node>& filters,
const Strides& strides,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const Strides& dilations,
const op::PadType& auto_pad,
const int64_t group,
const int64_t deformable_group,
const bool bilinear_interpolation_pad)
: DeformableConvolutionBase({arg, offsets, filters},
strides,
pads_begin,
pads_end,
dilations,
auto_pad,
group,
deformable_group)
, m_bilinear_interpolation_pad(bilinear_interpolation_pad)
{
constructor_validate_and_infer_types();
}
op::v8::DeformableConvolution::DeformableConvolution(const Output<Node>& arg,
const Output<Node>& offsets,
const Output<Node>& filters,
const Output<Node>& mask,
const Strides& strides,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const Strides& dilations,
const op::PadType& auto_pad,
const int64_t group,
const int64_t deformable_group,
const bool bilinear_interpolation_pad)
: DeformableConvolutionBase({arg, offsets, filters, mask},
strides,
pads_begin,
pads_end,
dilations,
auto_pad,
group,
deformable_group)
, m_bilinear_interpolation_pad(bilinear_interpolation_pad)
{
constructor_validate_and_infer_types();
}
bool op::v8::DeformableConvolution::visit_attributes(AttributeVisitor& visitor)
{
NGRAPH_OP_SCOPE(DeformableConvolution_v8_visit_attributes);
visitor.on_attribute("bilinear_interpolation_pad", m_bilinear_interpolation_pad);
return DeformableConvolutionBase::visit_attributes(visitor);
}
void op::v8::DeformableConvolution::validate_and_infer_types()
{
NGRAPH_OP_SCOPE(DeformableConvolution_v8_validate_and_infer_types);
DeformableConvolutionBase::validate_and_infer_types();
if (inputs().size() == 4)
{
const PartialShape& data_pshape = get_input_partial_shape(0);
const PartialShape& filters_pshape = get_input_partial_shape(2);
const PartialShape& mask_pshape = get_input_partial_shape(3);
element::Type mask_et = get_input_element_type(3);
NODE_VALIDATION_CHECK(this,
mask_et.is_real() || mask_et.is_integral_number(),
"Element type of Mask input must be numeric. Got: ",
mask_et);
NODE_VALIDATION_CHECK(this,
mask_pshape.rank().compatible(4),
"Mask input must be of rank 4. Got: ",
mask_pshape.rank());
if (mask_pshape.rank().is_static() && mask_pshape[1].is_static())
{
if (filters_pshape.rank().is_static() && filters_pshape[2].is_static() &&
filters_pshape[3].is_static())
{
auto offsets_channels = m_deformable_group * filters_pshape[2].get_length() *
filters_pshape[3].get_length();
NODE_VALIDATION_CHECK(this,
mask_pshape[1].get_length() == offsets_channels,
"The channels dimension of mask input is not "
"compatible with filters and 'deformable group' attribute. "
"Mask input shape: ",
mask_pshape,
", deformable 'group' attribute value: ",
m_deformable_group,
", filters shape: ",
filters_pshape);
}
// At least we can check if mask channels is evenly divisible by deformable
// group attribute
NODE_VALIDATION_CHECK(this,
mask_pshape[1].get_length() % m_deformable_group == 0,
"The channels dimension of mask input must be "
"evenly divisible by the 'deformable group' value along the "
"channels axis. Offsets input shape: ",
mask_pshape,
", 'deformable group' attribute value: ",
m_deformable_group);
if (data_pshape.rank().is_static())
{
NODE_VALIDATION_CHECK(
this,
mask_pshape[0].compatible(data_pshape[0]),
"Data batch and mask batch dimension must be same value. Got: ",
mask_pshape[0],
" and ",
data_pshape[0]);
}
}
PartialShape result_pshape = get_output_partial_shape(0);
if (result_pshape.rank().is_static() && mask_pshape.rank().is_static())
{
NODE_VALIDATION_CHECK(this,
result_pshape[2].compatible(mask_pshape[2]) &&
result_pshape[3].compatible(mask_pshape[3]),
"Spatial dimensions of mask and output must be equal. Got: ",
mask_pshape[2],
", ",
mask_pshape[3],
" and ",
result_pshape[2],
", ",
result_pshape[3]);
}
}
}
std::shared_ptr<Node>
op::v8::DeformableConvolution::clone_with_new_inputs(const OutputVector& new_args) const
{
NGRAPH_OP_SCOPE(DeformableConvolution_v8_clone_with_new_inputs);
check_new_args_count(this, new_args);
NODE_VALIDATION_CHECK(
this, new_args.size() >= 3 && new_args.size() <= 4, "Number of inputs must be 3 or 4");
switch (new_args.size())
{
case 3:
return std::make_shared<DeformableConvolution>(new_args.at(0),
new_args.at(1),
new_args.at(2),
m_strides,
m_pads_begin,
m_pads_end,
m_dilations,
m_auto_pad,
m_group,
m_deformable_group,
m_bilinear_interpolation_pad);
default:
return std::make_shared<DeformableConvolution>(new_args.at(0),
new_args.at(1),
new_args.at(2),
new_args.at(3),
m_strides,
m_pads_begin,
m_pads_end,
m_dilations,
m_auto_pad,
m_group,
m_deformable_group,
m_bilinear_interpolation_pad);
}
}
op::v1::DeformableConvolution::DeformableConvolution(const Output<Node>& arg,
const Output<Node>& offsets,
@ -22,218 +202,34 @@ op::v1::DeformableConvolution::DeformableConvolution(const Output<Node>& arg,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const Strides& dilations,
const PadType& auto_pad,
const op::PadType& auto_pad,
const int64_t group,
const int64_t deformable_group)
: Op({arg, offsets, filters})
, m_strides(strides)
, m_dilations(dilations)
, m_pads_begin(pads_begin)
, m_pads_end(pads_end)
, m_auto_pad(auto_pad)
, m_group(group)
, m_deformable_group(deformable_group)
: DeformableConvolutionBase({arg, offsets, filters},
strides,
pads_begin,
pads_end,
dilations,
auto_pad,
group,
deformable_group)
{
constructor_validate_and_infer_types();
}
bool op::v1::DeformableConvolution::visit_attributes(AttributeVisitor& visitor)
{
NGRAPH_OP_SCOPE(v1_DeformableConvolution_visit_attributes);
visitor.on_attribute("strides", m_strides);
visitor.on_attribute("dilations", m_dilations);
visitor.on_attribute("pads_begin", m_pads_begin);
visitor.on_attribute("pads_end", m_pads_end);
visitor.on_attribute("auto_pad", m_auto_pad);
visitor.on_attribute("group", m_group);
visitor.on_attribute("deformable_group", m_deformable_group);
return true;
}
void op::v1::DeformableConvolution::validate_and_infer_types()
{
NGRAPH_OP_SCOPE(v1_DeformableConvolution_validate_and_infer_types);
const PartialShape& data_batch_pshape = get_input_partial_shape(0);
const PartialShape& offsets_pshape = get_input_partial_shape(1);
const PartialShape& filters_pshape = get_input_partial_shape(2);
element::Type data_batch_et = get_input_element_type(0);
element::Type offsets_et = get_input_element_type(1);
element::Type filters_et = get_input_element_type(2);
element::Type result_et;
NODE_VALIDATION_CHECK(this,
element::Type::merge(result_et, data_batch_et, offsets_et) &&
element::Type::merge(result_et, result_et, filters_et),
"Element types of inputs do not match. Got: data batch (",
data_batch_et,
"), offsets (",
offsets_et,
") and filters (",
filters_et,
")");
NODE_VALIDATION_CHECK(this,
result_et.is_real() || result_et.is_integral_number(),
"Element type of inputs must be numeric. Got: ",
result_et);
Rank result_ps_rank{};
NODE_VALIDATION_CHECK(
this,
Rank::merge(result_ps_rank, data_batch_pshape.rank(), offsets_pshape.rank()) &&
Rank::merge(result_ps_rank, result_ps_rank, filters_pshape.rank()),
"Ranks of inputs do not match. Got: data batch shape ",
data_batch_pshape,
", offsets shape ",
offsets_pshape,
", filters shape ",
filters_pshape);
NODE_VALIDATION_CHECK(
this, result_ps_rank.compatible(4), "Inputs must be of rank 4. Got: ", result_ps_rank);
NODE_VALIDATION_CHECK(
this, m_group > 0, "Attribute 'group' must be any value starting from 1. Got: ", m_group);
NODE_VALIDATION_CHECK(this,
m_deformable_group > 0,
"Attribute 'deformable group' must be any value starting from 1. Got: ",
m_deformable_group);
if (offsets_pshape.rank().is_static())
{
if (offsets_pshape[1].is_static())
{
if (filters_pshape.rank().is_static() && filters_pshape[2].is_static() &&
filters_pshape[3].is_static())
{
auto offsets_channels = m_deformable_group * filters_pshape[2].get_length() *
filters_pshape[3].get_length() * 2;
NODE_VALIDATION_CHECK(this,
offsets_pshape[1].get_length() == offsets_channels,
"The channels dimension of offsets input is not "
"compatible with filters and 'deformable group' attribute. "
"Offsets input shape: ",
offsets_pshape,
", deformable 'group' attribute value: ",
m_deformable_group,
", filters shape: ",
filters_pshape);
}
else
{
// At least we can check if offsets channels is evenly divisible by deformable
// group attribute
NODE_VALIDATION_CHECK(this,
offsets_pshape[1].get_length() % m_deformable_group == 0,
"The channels dimension of offsets input must be "
"evenly divisible by the 'deformable group' value along the "
"channels axis. Offsets input shape: ",
offsets_pshape,
", 'deformable group' attribute value: ",
m_deformable_group);
}
}
if (data_batch_pshape.rank().is_static())
{
NODE_VALIDATION_CHECK(
this,
offsets_pshape[0].compatible(data_batch_pshape[0]),
"Data batch and offsets batch dimension must be same value. Got: ",
offsets_pshape[0],
" and ",
data_batch_pshape[0]);
}
}
if (data_batch_pshape.rank().is_static() && data_batch_pshape[1].is_static())
{
NODE_VALIDATION_CHECK(this,
data_batch_pshape[1].get_length() % m_group == 0,
"The input data shape must be evenly divisible by the 'group' value "
"along the channels axis. Current input shape: ",
data_batch_pshape,
", 'group' attribute value: ",
m_group);
}
if (filters_pshape.rank().is_static() && filters_pshape[0].is_static())
{
NODE_VALIDATION_CHECK(
this,
filters_pshape[0].get_length() % m_group == 0,
"The filters shape must be evenly divisible by the 'group' value along "
"the channels axis. Current filters shape: ",
filters_pshape,
", 'group' attribute value: ",
m_group);
}
// adjust filter shape to reuse regular infer_convolution_forward()
const auto new_filters_pshape = [&](int groups) {
auto new_shape(filters_pshape);
if (new_shape.rank().is_static())
{
new_shape[1] *= groups;
}
return new_shape;
}(m_group);
PartialShape result_shape =
validate_and_infer_convolution_forward_output_shape(this,
result_ps_rank,
data_batch_pshape,
new_filters_pshape,
m_auto_pad,
m_strides,
m_dilations,
m_pads_begin,
m_pads_end);
if (result_shape.rank().is_static() && offsets_pshape.rank().is_static())
{
PartialShape result_spatial_shape = [&result_shape]() {
vector<Dimension> result_spatial_dims{result_shape};
result_spatial_dims.erase(result_spatial_dims.begin(), result_spatial_dims.begin() + 2);
return PartialShape{result_spatial_dims};
}();
PartialShape offsets_spatial_shape = [&offsets_pshape]() {
vector<Dimension> offsets_spatial_dims{offsets_pshape};
offsets_spatial_dims.erase(offsets_spatial_dims.begin(),
offsets_spatial_dims.begin() + 2);
return PartialShape{offsets_spatial_dims};
}();
NODE_VALIDATION_CHECK(this,
offsets_spatial_shape.compatible(result_spatial_shape),
"Spatial dimensions of offsets and output must be equal. Got: ",
offsets_spatial_shape,
" and ",
result_spatial_shape);
if (result_shape[0].is_dynamic())
{
result_shape[0] = offsets_pshape[0]; // batch size
}
}
set_output_type(0, result_et, result_shape);
}
shared_ptr<Node>
std::shared_ptr<Node>
op::v1::DeformableConvolution::clone_with_new_inputs(const OutputVector& new_args) const
{
NGRAPH_OP_SCOPE(v1_DeformableConvolution_clone_with_new_inputs);
NGRAPH_OP_SCOPE(DeformableConvolution_v1_clone_with_new_inputs);
check_new_args_count(this, new_args);
return make_shared<v1::DeformableConvolution>(new_args.at(0),
new_args.at(1),
new_args.at(2),
m_strides,
m_pads_begin,
m_pads_end,
m_dilations,
m_auto_pad,
m_group,
m_deformable_group);
return std::make_shared<DeformableConvolution>(new_args.at(0),
new_args.at(1),
new_args.at(2),
m_strides,
m_pads_begin,
m_pads_end,
m_dilations,
m_auto_pad,
m_group,
m_deformable_group);
}

View File

@ -0,0 +1,219 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "ngraph/op/util/deformable_convolution_base.hpp"
#include "itt.hpp"
#include "ngraph/axis_vector.hpp"
#include "ngraph/coordinate_diff.hpp"
#include "ngraph/op/reshape.hpp"
#include "ngraph/util.hpp"
#include "ngraph/validation_util.hpp"
using namespace std;
using namespace ngraph;
NGRAPH_RTTI_DEFINITION(op::util::DeformableConvolutionBase, "DeformableConvolutionBase", 0);
op::util::DeformableConvolutionBase::DeformableConvolutionBase(const OutputVector& arguments,
const Strides& strides,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const Strides& dilations,
const PadType& auto_pad,
const int64_t group,
const int64_t deformable_group)
: Op(arguments)
, m_strides(strides)
, m_dilations(dilations)
, m_pads_begin(pads_begin)
, m_pads_end(pads_end)
, m_auto_pad(auto_pad)
, m_group(group)
, m_deformable_group(deformable_group)
{
}
bool op::util::DeformableConvolutionBase::visit_attributes(AttributeVisitor& visitor)
{
NGRAPH_OP_SCOPE(util_DeformableConvolutionBase_visit_attributes);
visitor.on_attribute("strides", m_strides);
visitor.on_attribute("dilations", m_dilations);
visitor.on_attribute("pads_begin", m_pads_begin);
visitor.on_attribute("pads_end", m_pads_end);
visitor.on_attribute("auto_pad", m_auto_pad);
visitor.on_attribute("group", m_group);
visitor.on_attribute("deformable_group", m_deformable_group);
return true;
}
void op::util::DeformableConvolutionBase::validate_and_infer_types()
{
NGRAPH_OP_SCOPE(util_DeformableConvolutionBase_validate_and_infer_types);
const PartialShape& data_batch_pshape = get_input_partial_shape(0);
const PartialShape& offsets_pshape = get_input_partial_shape(1);
const PartialShape& filters_pshape = get_input_partial_shape(2);
element::Type data_batch_et = get_input_element_type(0);
element::Type offsets_et = get_input_element_type(1);
element::Type filters_et = get_input_element_type(2);
element::Type result_et;
NODE_VALIDATION_CHECK(this,
element::Type::merge(result_et, data_batch_et, offsets_et) &&
element::Type::merge(result_et, result_et, filters_et),
"Element types of inputs do not match. Got: data batch (",
data_batch_et,
"), offsets (",
offsets_et,
") and filters (",
filters_et,
")");
NODE_VALIDATION_CHECK(this,
result_et.is_real() || result_et.is_integral_number(),
"Element type of inputs must be numeric. Got: ",
result_et);
Rank result_ps_rank{};
NODE_VALIDATION_CHECK(
this,
Rank::merge(result_ps_rank, data_batch_pshape.rank(), offsets_pshape.rank()) &&
Rank::merge(result_ps_rank, result_ps_rank, filters_pshape.rank()),
"Ranks of inputs do not match. Got: data batch shape ",
data_batch_pshape,
", offsets shape ",
offsets_pshape,
", filters shape ",
filters_pshape);
NODE_VALIDATION_CHECK(
this, result_ps_rank.compatible(4), "Inputs must be of rank 4. Got: ", result_ps_rank);
NODE_VALIDATION_CHECK(
this, m_group > 0, "Attribute 'group' must be any value starting from 1. Got: ", m_group);
NODE_VALIDATION_CHECK(this,
m_deformable_group > 0,
"Attribute 'deformable group' must be any value starting from 1. Got: ",
m_deformable_group);
if (offsets_pshape.rank().is_static())
{
if (offsets_pshape[1].is_static())
{
if (filters_pshape.rank().is_static() && filters_pshape[2].is_static() &&
filters_pshape[3].is_static())
{
auto offsets_channels = m_deformable_group * filters_pshape[2].get_length() *
filters_pshape[3].get_length() * 2;
NODE_VALIDATION_CHECK(this,
offsets_pshape[1].get_length() == offsets_channels,
"The channels dimension of offsets input is not "
"compatible with filters and 'deformable group' attribute. "
"Offsets input shape: ",
offsets_pshape,
", deformable 'group' attribute value: ",
m_deformable_group,
", filters shape: ",
filters_pshape);
}
else
{
// At least we can check if offsets channels is evenly divisible by deformable
// group attribute
NODE_VALIDATION_CHECK(this,
offsets_pshape[1].get_length() % m_deformable_group == 0,
"The channels dimension of offsets input must be "
"evenly divisible by the 'deformable group' value along the "
"channels axis. Offsets input shape: ",
offsets_pshape,
", 'deformable group' attribute value: ",
m_deformable_group);
}
}
if (data_batch_pshape.rank().is_static())
{
NODE_VALIDATION_CHECK(
this,
offsets_pshape[0].compatible(data_batch_pshape[0]),
"Data batch and offsets batch dimension must be same value. Got: ",
offsets_pshape[0],
" and ",
data_batch_pshape[0]);
}
}
if (data_batch_pshape.rank().is_static() && data_batch_pshape[1].is_static())
{
NODE_VALIDATION_CHECK(this,
data_batch_pshape[1].get_length() % m_group == 0,
"The input data shape must be evenly divisible by the 'group' value "
"along the channels axis. Current input shape: ",
data_batch_pshape,
", 'group' attribute value: ",
m_group);
}
if (filters_pshape.rank().is_static() && filters_pshape[0].is_static())
{
NODE_VALIDATION_CHECK(
this,
filters_pshape[0].get_length() % m_group == 0,
"The filters shape must be evenly divisible by the 'group' value along "
"the channels axis. Current filters shape: ",
filters_pshape,
", 'group' attribute value: ",
m_group);
}
// adjust filter shape to reuse regular infer_convolution_forward()
const auto new_filters_pshape = [&](int groups) {
auto new_shape(filters_pshape);
if (new_shape.rank().is_static())
{
new_shape[1] *= groups;
}
return new_shape;
}(m_group);
PartialShape result_shape =
validate_and_infer_convolution_forward_output_shape(this,
result_ps_rank,
data_batch_pshape,
new_filters_pshape,
m_auto_pad,
m_strides,
m_dilations,
m_pads_begin,
m_pads_end);
if (result_shape.rank().is_static() && offsets_pshape.rank().is_static())
{
PartialShape result_spatial_shape = [&result_shape]() {
vector<Dimension> result_spatial_dims{result_shape};
result_spatial_dims.erase(result_spatial_dims.begin(), result_spatial_dims.begin() + 2);
return PartialShape{result_spatial_dims};
}();
PartialShape offsets_spatial_shape = [&offsets_pshape]() {
vector<Dimension> offsets_spatial_dims{offsets_pshape};
offsets_spatial_dims.erase(offsets_spatial_dims.begin(),
offsets_spatial_dims.begin() + 2);
return PartialShape{offsets_spatial_dims};
}();
NODE_VALIDATION_CHECK(this,
offsets_spatial_shape.compatible(result_spatial_shape),
"Spatial dimensions of offsets and output must be equal. Got: ",
offsets_spatial_shape,
" and ",
result_spatial_shape);
if (result_shape[0].is_dynamic())
{
result_shape[0] = offsets_pshape[0]; // batch size
}
}
set_output_type(0, result_et, result_shape);
}

View File

@ -115,6 +115,7 @@ set(SRC
type_prop/ctc_greedy_decoder_seq_len.cpp
type_prop/ctc_loss.cpp
type_prop/deformable_convolution.cpp
type_prop/deformable_convolution_opset8.cpp
type_prop/deformable_psroi_pooling.cpp
type_prop/detection_output.cpp
type_prop/depth_to_space.cpp
@ -234,6 +235,7 @@ set(SRC
visitors/op/convolution_backprop.cpp
visitors/op/cos.cpp
visitors/op/cum_sum.cpp
visitors/op/deformable_convolution.cpp
visitors/op/deformable_psroi_pooling.cpp
visitors/op/depth_to_space.cpp
visitors/op/detection_output.cpp

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,76 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "gtest/gtest.h"
#include "ngraph/ngraph.hpp"
#include "ngraph/op/util/attr_types.hpp"
#include "ngraph/opsets/opset8.hpp"
#include "util/visitor.hpp"
using namespace std;
using namespace ngraph;
using ngraph::test::NodeBuilder;
using ngraph::test::ValueMap;
TEST(attributes, deformable_convolution_default_attributes)
{
NodeBuilder::get_ops().register_factory<opset8::DeformableConvolution>();
const Shape inputs_shape{1, 1, 5, 5};
auto data = make_shared<op::Parameter>(element::f32, Shape{1, 1, 5, 5});
auto filters = make_shared<op::Parameter>(element::f32, Shape{1, 1, 3, 3});
auto offsets = make_shared<op::Parameter>(element::f32, Shape{1, 18, 3, 3});
auto strides = Strides{1, 1};
auto pads_begin = CoordinateDiff{0, 0};
auto pads_end = CoordinateDiff{0, 0};
auto dilations = Strides{1, 1};
auto convolution = make_shared<opset8::DeformableConvolution>(data, offsets, filters, strides, pads_begin, pads_end, dilations);
NodeBuilder builder(convolution);
auto g_convolution = as_type_ptr<opset8::DeformableConvolution>(builder.create());
// attribute count
const auto expected_attr_count = 8;
EXPECT_EQ(builder.get_value_map_size(), expected_attr_count);
EXPECT_EQ(g_convolution->get_strides(), convolution->get_strides());
EXPECT_EQ(g_convolution->get_pads_begin(), convolution->get_pads_begin());
EXPECT_EQ(g_convolution->get_pads_end(), convolution->get_pads_end());
EXPECT_EQ(g_convolution->get_dilations(), convolution->get_dilations());
EXPECT_EQ(g_convolution->get_auto_pad(), convolution->get_auto_pad());
EXPECT_EQ(g_convolution->get_group(), convolution->get_group());
EXPECT_EQ(g_convolution->get_deformable_group(), convolution->get_deformable_group());
EXPECT_EQ(g_convolution->get_bilinear_interpolation_pad(), convolution->get_bilinear_interpolation_pad());
}
TEST(attributes, deformable_convolution_attributes)
{
NodeBuilder::get_ops().register_factory<opset8::DeformableConvolution>();
const Shape inputs_shape{1, 1, 5, 5};
auto data = make_shared<op::Parameter>(element::f32, Shape{1, 2, 5, 5});
auto filters = make_shared<op::Parameter>(element::f32, Shape{2, 1, 3, 3});
auto offsets = make_shared<op::Parameter>(element::f32, Shape{1, 36, 5, 5});
auto mask = make_shared<op::Parameter>(element::f32, Shape{1, 18, 5, 5});
auto strides = Strides{1, 1};
auto pads_begin = CoordinateDiff{0, 0};
auto pads_end = CoordinateDiff{0, 0};
auto dilations = Strides{1, 1};
auto convolution = make_shared<opset8::DeformableConvolution>(data, offsets, filters, mask, strides, pads_begin, pads_end, dilations,
op::PadType::SAME_LOWER, 2, 2, true);
NodeBuilder builder(convolution);
auto g_convolution = as_type_ptr<opset8::DeformableConvolution>(builder.create());
// attribute count
const auto expected_attr_count = 8;
EXPECT_EQ(builder.get_value_map_size(), expected_attr_count);
EXPECT_EQ(g_convolution->get_strides(), convolution->get_strides());
EXPECT_EQ(g_convolution->get_pads_begin(), convolution->get_pads_begin());
EXPECT_EQ(g_convolution->get_pads_end(), convolution->get_pads_end());
EXPECT_EQ(g_convolution->get_dilations(), convolution->get_dilations());
EXPECT_EQ(g_convolution->get_auto_pad(), convolution->get_auto_pad());
EXPECT_EQ(g_convolution->get_group(), convolution->get_group());
EXPECT_EQ(g_convolution->get_deformable_group(), convolution->get_deformable_group());
EXPECT_EQ(g_convolution->get_bilinear_interpolation_pad(), convolution->get_bilinear_interpolation_pad());
}