IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[OV20] Preprocessing: reverse_channels and RGB<->BGR conversion (#8098)

Browse Source 
* Initial draft

* Support of dynamic shapes
This commit is contained in:
Mikhail Nosov 2021-10-21 11:22:43 +03:00 committed by GitHub
parent 55a0e8332b
commit d39fe50470
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 226 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp
View File

@ -746,6 +746,101 @@ static RefPreprocessParams pre_and_post_processing() {
return res;
}
static RefPreprocessParams rgb_to_bgr() {
RefPreprocessParams res("rgb_to_bgr");
res.function = []() {
auto f = create_simple_function(element::f32, Shape{2, 1, 1, 3});
f = PrePostProcessor().input(InputInfo()
.tensor(InputTensorInfo().set_color_format(ColorFormat::RGB))
.preprocess(PreProcessSteps().convert_color(ColorFormat::BGR))).build(f);
return f;
};
res.inputs.emplace_back(Shape{2, 3, 1, 1}, element::f32, std::vector<float>{1, 2, 3, 4, 5, 6});
res.expected.emplace_back(Shape{2, 3, 1, 1}, element::f32, std::vector<float>{3, 2, 1, 6, 5, 4});
return res;
}
static RefPreprocessParams bgr_to_rgb() {
RefPreprocessParams res("bgr_to_rgb");
res.function = []() {
auto f = create_simple_function(element::f32, Shape{2, 1, 1, 3});
f = PrePostProcessor().input(InputInfo()
.tensor(InputTensorInfo().set_color_format(ColorFormat::BGR))
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB))).build(f);
return f;
};
res.inputs.emplace_back(Shape{2, 3, 1, 1}, element::f32, std::vector<float>{1, 2, 3, 4, 5, 6});
res.expected.emplace_back(Shape{2, 3, 1, 1}, element::f32, std::vector<float>{3, 2, 1, 6, 5, 4});
return res;
}
static RefPreprocessParams reverse_channels_nchw() {
RefPreprocessParams res("reverse_channels_nchw");
res.function = []() {
auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 2});
f = PrePostProcessor().input(InputInfo()
.tensor(InputTensorInfo().set_layout("NCHW"))
.preprocess(PreProcessSteps().reverse_channels())).build(f);
return f;
};
res.inputs.emplace_back(Shape{1, 2, 2, 2}, element::f32, std::vector<float>{1, 2, 3, 4, 5, 6, 7, 8});
res.expected.emplace_back(Shape{1, 2, 2, 2}, element::f32, std::vector<float>{5, 6, 7, 8, 1, 2, 3, 4});
return res;
}
static RefPreprocessParams reverse_channels_dyn_layout() {
RefPreprocessParams res("reverse_channels_dyn_layout");
res.function = []() {
auto f = create_simple_function(element::f32, PartialShape{1, 1, 3, 2});
f = PrePostProcessor().input(InputInfo()
.tensor(InputTensorInfo().set_color_format(ColorFormat::BGR).set_layout("...CN"))
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB))).build(f);
return f;
};
res.inputs.emplace_back(Shape{1, 1, 3, 2}, element::f32, std::vector<float>{1, 2, 3, 4, 5, 6});
res.expected.emplace_back(Shape{1, 1, 3, 2}, element::f32, std::vector<float>{5, 6, 3, 4, 1, 2});
return res;
}
static RefPreprocessParams reverse_dyn_shape() {
RefPreprocessParams res("reverse_dyn_shape");
res.function = []() {
auto f = create_simple_function(element::u8, PartialShape{Dimension::dynamic(),
Dimension::dynamic(),
Dimension::dynamic(),
Dimension::dynamic()});
f = PrePostProcessor().input(InputInfo()
.tensor(InputTensorInfo().set_layout("NCHW"))
.preprocess(PreProcessSteps().reverse_channels())).build(f);
return f;
};
res.inputs.emplace_back(element::u8, Shape{2, 2, 1, 3}, std::vector<uint8_t>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
res.expected.emplace_back(Shape{2, 2, 1, 3}, element::u8, std::vector<uint8_t>{4, 5, 6, 1, 2, 3, 10, 11, 12, 7, 8, 9});
return res;
}
static RefPreprocessParams reverse_fully_dyn_shape() {
RefPreprocessParams res("reverse_fully_dyn_shape");
res.function = []() {
auto f = create_simple_function(element::u8, PartialShape::dynamic());
auto p = PreProcessSteps();
p.reverse_channels();
f = PrePostProcessor().input(InputInfo()
.tensor(InputTensorInfo().set_layout("...C??"))
.preprocess(std::move(p))).build(f);
return f;
};
res.inputs.emplace_back(element::u8, Shape{2, 2, 1, 3}, std::vector<uint8_t>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
res.expected.emplace_back(Shape{2, 2, 1, 3}, element::u8, std::vector<uint8_t>{4, 5, 6, 1, 2, 3, 10, 11, 12, 7, 8, 9});
return res;
}
std::vector<RefPreprocessParams> allPreprocessTests() {
return std::vector<RefPreprocessParams> {
simple_mean_scale(),
@ -773,7 +868,13 @@ std::vector<RefPreprocessParams> allPreprocessTests() {
convert_color_nv12_layout_resize(),
element_type_before_convert_color_nv12(),
postprocess_2_inputs_basic(),
pre_and_post_processing()
pre_and_post_processing(),
rgb_to_bgr(),
bgr_to_rgb(),
reverse_channels_nchw(),
reverse_channels_dyn_layout(),
reverse_dyn_shape(),
reverse_fully_dyn_shape()
};
}

21
ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp
View File

@ -228,6 +228,27 @@ public:
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
PreProcessSteps&& convert_layout(const Layout& dst_layout = {}) &&;
/// \brief Reverse channels operation - Lvalue version.
///
/// \details Adds appropriate operation which reverses channels layout. Operation requires layout having 'C'
/// dimension Operation convert_color (RGB<->BGR) does reversing of channels also, but only for NHWC layout
///
/// \example Example: when user data has 'NCHW' layout (example is [1, 3, 224, 224] RGB order) but network expects
/// BGR planes order. Preprocessing may look like this:
///
/// \code{.cpp} auto proc =
/// PrePostProcessor()
/// .input(InputInfo()
/// .tensor(InputTensorInfo().set_layout("NCHW")) // User data is NCHW
/// .preprocess(PreProcessSteps()
/// .reverse_channels()
/// );
/// \endcode
///
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
PreProcessSteps& reverse_channels() &;
PreProcessSteps&& reverse_channels() &&;
};
} // namespace preprocess

4
ngraph/core/src/preprocess/color_utils.cpp
View File

@ -15,6 +15,10 @@ std::unique_ptr<ColorFormatInfo> ColorFormatInfo::get(ColorFormat format) {
case ColorFormat::NV12_TWO_PLANES:
res.reset(new ColorFormatInfoNV12_TwoPlanes(format));
break;
case ColorFormat::RGB:
case ColorFormat::BGR:
res.reset(new ColorFormatNHWC(format));
break;
default:
res.reset(new ColorFormatInfo(format));
break;

25
ngraph/core/src/preprocess/color_utils.hpp
View File

@ -70,9 +70,18 @@ protected:
};
// --- Derived classes ---
class ColorFormatInfoNV12_Single : public ColorFormatInfo {
class ColorFormatNHWC : public ColorFormatInfo {
public:
explicit ColorFormatInfoNV12_Single(ColorFormat format) : ColorFormatInfo(format) {}
explicit ColorFormatNHWC(ColorFormat format) : ColorFormatInfo(format) {}
Layout default_layout() const override {
return "NHWC";
}
};
class ColorFormatInfoNV12_Single : public ColorFormatNHWC {
public:
explicit ColorFormatInfoNV12_Single(ColorFormat format) : ColorFormatNHWC(format) {}
protected:
PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override {
@ -85,15 +94,11 @@ protected:
}
return result;
}
Layout default_layout() const override {
return "NHWC";
}
};
class ColorFormatInfoNV12_TwoPlanes : public ColorFormatInfo {
class ColorFormatInfoNV12_TwoPlanes : public ColorFormatNHWC {
public:
explicit ColorFormatInfoNV12_TwoPlanes(ColorFormat format) : ColorFormatInfo(format) {}
explicit ColorFormatInfoNV12_TwoPlanes(ColorFormat format) : ColorFormatNHWC(format) {}
size_t planes_count() const override {
return 2;
@ -119,10 +124,6 @@ protected:
}
return result;
}
Layout default_layout() const override {
return "NHWC";
}
};
} // namespace preprocess

10
ngraph/core/src/preprocess/pre_post_process.cpp
View File

@ -790,6 +790,16 @@ PreProcessSteps&& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_c
return std::move(*this);
}
PreProcessSteps& PreProcessSteps::reverse_channels() & {
m_impl->add_reverse_channels();
return *this;
}
PreProcessSteps&& PreProcessSteps::reverse_channels() && {
m_impl->add_reverse_channels();
return std::move(*this);
}
// --------------------- OutputTensorInfo ------------------
OutputTensorInfo::OutputTensorInfo() : m_impl(std::unique_ptr<OutputTensorInfoImpl>(new OutputTensorInfoImpl())) {}
OutputTensorInfo::OutputTensorInfo(OutputTensorInfo&&) noexcept = default;

52
ngraph/core/src/preprocess/preprocess_steps_impl.cpp
View File

@ -178,9 +178,9 @@ void PreStepsList::add_convert_layout_impl(const Layout& layout) {
}
void PreStepsList::add_convert_color_impl(const ColorFormat& dst_format) {
m_actions.emplace_back([&, dst_format](const std::vector<Output<Node>>& nodes,
const std::shared_ptr<Function>& function,
PreprocessingContext& context) {
m_actions.emplace_back([dst_format](const std::vector<Output<Node>>& nodes,
const std::shared_ptr<Function>& function,
PreprocessingContext& context) {
if (context.color_format() == dst_format) {
return std::make_tuple(nodes, false);
}
@ -221,6 +221,12 @@ void PreStepsList::add_convert_color_impl(const ColorFormat& dst_format) {
context.color_format() = dst_format;
return std::make_tuple(std::vector<Output<Node>>{convert}, true);
}
if ((context.color_format() == ColorFormat::RGB || context.color_format() == ColorFormat::BGR) &&
(dst_format == ColorFormat::RGB || dst_format == ColorFormat::BGR)) {
auto res = reverse_channels(nodes, function, context);
context.color_format() = dst_format;
return res;
}
OPENVINO_ASSERT(false,
"Source color format '",
color_format_name(context.color_format()),
@ -228,6 +234,46 @@ void PreStepsList::add_convert_color_impl(const ColorFormat& dst_format) {
});
}
void PreStepsList::add_reverse_channels() {
m_actions.emplace_back([](const std::vector<Output<Node>>& nodes,
const std::shared_ptr<Function>& function,
PreprocessingContext& context) {
return reverse_channels(nodes, function, context);
});
}
std::tuple<std::vector<Output<Node>>, bool> PreStepsList::reverse_channels(const std::vector<Output<Node>>& nodes,
const std::shared_ptr<Function>& function,
PreprocessingContext& context) {
OPENVINO_ASSERT(nodes.size() == 1, "Internal error: can't reverse channels for multi-plane inputs");
OPENVINO_ASSERT(ov::layout::has_channels(context.layout()),
"Layout ",
context.layout().to_string(),
" doesn't have `channels` dimension");
auto channels_idx = ov::layout::channels(context.layout());
// Get shape of user's input tensor (e.g. Tensor[1, 3, 224, 224] -> {1, 3, 224, 224})
auto shape_of = std::make_shared<ov::op::v0::ShapeOf>(nodes[0]); // E.g. {1, 3, 224, 224}
auto constant_chan_idx = op::v0::Constant::create(element::i32, {}, {channels_idx}); // E.g. 1
auto constant_chan_axis = op::v0::Constant::create(element::i32, {}, {0});
// Gather will return scalar with number of channels (e.g. 3)
auto gather_channels_num = std::make_shared<op::v8::Gather>(shape_of, constant_chan_idx, constant_chan_axis);
// Create Range from channels_num-1 to 0 (e.g. {2, 1, 0})
auto const_minus1 = op::v0::Constant::create(element::i64, {}, {-1});
auto channels_num_minus1 = std::make_shared<op::v1::Add>(gather_channels_num, const_minus1); // E.g. 3-1=2
// Add range
auto range_to = op::v0::Constant::create(element::i64, {}, {-1});
auto range_step = op::v0::Constant::create(element::i64, {}, {-1});
// E.g. {2, 1, 0}
auto range = std::make_shared<op::v4::Range>(channels_num_minus1, range_to, range_step, element::i32);
// Gather slices in reverse order (indexes are specified by 'range' operation)
auto constant_axis = op::v0::Constant::create(element::i32, {1}, {channels_idx});
auto convert = std::make_shared<op::v8::Gather>(nodes[0], range, constant_axis);
return std::make_tuple(std::vector<Output<Node>>{convert}, false);
}
//------------- Post processing ------
void PostStepsList::add_convert_impl(const element::Type& type) {
m_actions.emplace_back([type](const Output<Node>& node, PostprocessingContext& ctxt) {

6
ngraph/core/src/preprocess/preprocess_steps_impl.hpp
View File

@ -155,6 +155,7 @@ public:
void add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width);
void add_convert_layout_impl(const Layout& layout);
void add_convert_color_impl(const ColorFormat& dst_format);
void add_reverse_channels();
const std::list<InternalPreprocessOp>& actions() const {
return m_actions;
@ -163,6 +164,11 @@ public:
return m_actions;
}
private:
static std::tuple<std::vector<Output<Node>>, bool> reverse_channels(const std::vector<Output<Node>>& nodes,
const std::shared_ptr<Function>& function,
PreprocessingContext& context);
private:
std::list<InternalPreprocessOp> m_actions;
};

21
ngraph/test/preprocess.cpp
View File

@ -697,6 +697,27 @@ TEST(pre_post_process, preprocess_convert_layout_same) {
EXPECT_EQ(size_old, f->get_ordered_ops().size());
}
TEST(pre_post_process, preprocess_reverse_channels_multiple_planes) {
auto f = create_simple_function(element::f32, Shape{1, 3, 2, 2});
EXPECT_THROW(
f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES, {"Y", "UV"}))
.preprocess(PreProcessSteps().reverse_channels()))
.build(f),
ov::AssertFailure);
}
TEST(pre_post_process, preprocess_reverse_channels_no_c_dim) {
auto f = create_simple_function(element::f32, Shape{1, 3, 2, 2});
EXPECT_THROW(f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_layout("N?HW"))
.preprocess(PreProcessSteps().reverse_channels()))
.build(f),
ov::AssertFailure);
}
// --- PostProcess - set/convert element type ---
TEST(pre_post_process, postprocess_convert_element_type_explicit) {