IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[PP GAPI] Extended preprocessing graph to support precision conversions (#2290)

Browse Source 
- not yet visible via plugin interface
- for resize non U8 input is converted to  FP32
- tests
This commit is contained in:
Anton Potapov
2020-09-22 17:40:01 +03:00
committed by GitHub
parent 079bf2b092
commit f9c8b692fb
5 changed files with 155 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
inference-engine/src/preprocessing/ie_preprocess_gapi.cpp
View File

@@ -30,10 +30,12 @@
namespace InferenceEngine {
namespace {
int get_cv_depth(const TensorDesc &ie_desc);
namespace G {
struct Strides {int N; int C; int H; int W;};
struct Dims {int N; int C; int H; int W;};
struct Desc {Dims d; Strides s;};
struct Desc {Dims d; Strides s; int prec;};
void fix_strides_nhwc(const Dims &d, Strides &s) {
if (s.W > d.C) {
@@ -66,7 +68,7 @@ namespace G {
if (nhwc_layout) fix_strides_nhwc(d, s);
return Desc{d, s};
return Desc{d, s, get_cv_depth(ie_desc)};
}
Desc decompose(const Blob::Ptr& blob) {
@@ -78,6 +80,8 @@ inline int get_cv_depth(const TensorDesc &ie_desc) {
switch (ie_desc.getPrecision()) {
case Precision::U8: return CV_8U;
case Precision::FP32: return CV_32F;
case Precision::U16: return CV_16U;
default: THROW_IE_EXCEPTION << "Unsupported data type";
}
}
@@ -371,6 +375,7 @@ G::Desc getGDesc(G::Desc in_desc_y, const NV12Blob::Ptr &) {
auto nv12_desc = G::Desc{};
nv12_desc.d = in_desc_y.d;
nv12_desc.d.C = 2;
nv12_desc.prec = in_desc_y.prec;
return nv12_desc;
}
@@ -379,6 +384,7 @@ G::Desc getGDesc(G::Desc in_desc_y, const I420Blob::Ptr &) {
auto i420_desc = G::Desc{};
i420_desc.d = in_desc_y.d;
i420_desc.d.C = 3;
i420_desc.prec = in_desc_y.prec;
return i420_desc;
}
@@ -543,8 +549,7 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
Layout out_layout,
ResizeAlgorithm algorithm,
ColorFormat input_color_format,
ColorFormat output_color_format,
int precision) {
ColorFormat output_color_format) {
// perform basic validation to ensure our assumptions about input and output are correct
validateColorFormats(in_desc, out_desc, in_layout, out_layout, input_color_format,
output_color_format);
@@ -569,7 +574,7 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
(io_color_formats == std::make_tuple(ColorFormat::BGRX, ColorFormat::BGR));
const bool specific_case_of_preproc = ((in_layout == NHWC || specific_yuv420_input_handling)
&& (in_desc.d.C == 3 || specific_yuv420_input_handling || drop_channel)
&& (precision == CV_8U)
&& ((in_desc.prec == CV_8U) && (in_desc.prec == out_desc.prec))
&& (algorithm == RESIZE_BILINEAR)
&& (input_color_format == ColorFormat::RAW
|| input_color_format == output_color_format
@@ -591,9 +596,9 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
auto planes = drop_channel ?
to_vec(gapi::ScalePlanes4:: on(
color_converted_input[0], precision, input_sz, scale_sz, cv::INTER_LINEAR))
color_converted_input[0], in_desc.prec, input_sz, scale_sz, cv::INTER_LINEAR))
: to_vec(gapi::ScalePlanes ::on(
color_converted_input[0], precision, input_sz, scale_sz, cv::INTER_LINEAR));
color_converted_input[0], in_desc.prec, input_sz, scale_sz, cv::INTER_LINEAR));
if (drop_channel) {
planes.pop_back();
@@ -623,8 +628,13 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
<< number_of_planes << " != " << out_desc.d.C;
}
const int tmp_prec = CV_32F;
std::vector<cv::GMat> outputs;
if (algorithm != NO_RESIZE) {
const bool resize_needed = (algorithm != NO_RESIZE);
const bool need_tmp_prec_conv = resize_needed && (in_desc.prec != CV_8U) && (in_desc.prec != CV_32F);
if (resize_needed) {
// resize every plane
std::vector<cv::GMat> out_planes;
out_planes.reserve(planes.size());
@@ -635,18 +645,36 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
default: THROW_IE_EXCEPTION << "Unsupported resize operation";
}
} (algorithm);
const auto input_sz = cv::gapi::own::Size(in_desc.d.W, in_desc.d.H);
const auto scale_sz = cv::gapi::own::Size(out_desc.d.W, out_desc.d.H);
const auto scale_fcn = std::bind(&gapi::ScalePlane::on,
std::placeholders::_1,
precision,
input_sz, scale_sz, interp_type);
std::transform(planes.begin(), planes.end(), std::back_inserter(out_planes), scale_fcn);
std::transform(planes.begin(), planes.end(), std::back_inserter(out_planes), [&](const cv::GMat& m) {
const auto input_sz = cv::gapi::own::Size(in_desc.d.W, in_desc.d.H);
const auto scale_sz = cv::gapi::own::Size(out_desc.d.W, out_desc.d.H);
cv::GMat converted = m;
int prec = in_desc.prec;
if (need_tmp_prec_conv) {
std::tie(converted, prec) = std::make_tuple(gapi::ConvertDepth::on(m, tmp_prec), tmp_prec);
}
return gapi::ScalePlane::on(converted, prec, input_sz, scale_sz, interp_type);
});
outputs = out_planes;
} else {
outputs = planes;
}
if ((in_desc.prec != out_desc.prec) || need_tmp_prec_conv) {
auto convert_prec = [](const std::vector<cv::GMat> & src_gmats, int dst_precision) {
std::vector<cv::GMat> dst_gmats;
std::transform(src_gmats.begin(), src_gmats.end(), std::back_inserter(dst_gmats), [&](cv::GMat const& m){
return gapi::ConvertDepth::on(m, dst_precision);
});
return dst_gmats;
};
outputs = convert_prec(outputs, out_desc.prec);
}
// convert to interleaved if NHWC is required as output
if (out_layout == NHWC) {
outputs = merge(outputs, out_desc.d.C);
@@ -939,8 +967,7 @@ bool PreprocEngine::preprocessBlob(const BlobTypePtr &inBlob, MemoryBlob::Ptr &o
out_layout,
algorithm,
in_fmt,
out_fmt,
get_cv_depth(in_desc_ie)));
out_fmt));
}
}
@@ -953,7 +980,7 @@ bool PreprocEngine::preprocessBlob(const BlobTypePtr &inBlob, MemoryBlob::Ptr &o
return true;
}
bool PreprocEngine::preprocessWithGAPI(Blob::Ptr &inBlob, Blob::Ptr &outBlob,
bool PreprocEngine::preprocessWithGAPI(const Blob::Ptr &inBlob, Blob::Ptr &outBlob,
const ResizeAlgorithm& algorithm, ColorFormat in_fmt, bool omp_serial, int batch_size) {
if (!useGAPI()) {
return false;

2
inference-engine/src/preprocessing/ie_preprocess_gapi.hpp
View File

@@ -54,7 +54,7 @@ public:
static bool useGAPI();
static void checkApplicabilityGAPI(const Blob::Ptr &src, const Blob::Ptr &dst);
static int getCorrectBatchSize(int batch_size, const Blob::Ptr& roiBlob);
bool preprocessWithGAPI(Blob::Ptr &inBlob, Blob::Ptr &outBlob, const ResizeAlgorithm &algorithm,
bool preprocessWithGAPI(const Blob::Ptr &inBlob, Blob::Ptr &outBlob, const ResizeAlgorithm &algorithm,
ColorFormat in_fmt, bool omp_serial, int batch_size = -1);
};

96
inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.cpp
View File

@@ -23,6 +23,8 @@
#include <map>
#include <stdexcept>
#include <fluid_test_computations.hpp>
// Can be set externally (via CMake) if built with -DGAPI_TEST_PERF=ON
@@ -82,6 +84,7 @@ cv::String depthToString(int depth)
{
case CV_8U : return "CV_8U";
case CV_32F : return "CV_32F";
case CV_16U : return "CV_16U";
}
CV_Assert(!"ERROR: unsupported depth!");
return nullptr;
@@ -1078,14 +1081,17 @@ TEST_P(MergeTestIE, AccuracyTest)
TEST_P(PreprocTest, Performance)
{
using namespace InferenceEngine;
Precision prec;
std::pair<Precision, Precision> precisions;
ResizeAlgorithm interp;
Layout in_layout, out_layout;
std::pair<int, int> ocv_channels{-1, -1};
std::pair<cv::Size, cv::Size> sizes;
ColorFormat in_fmt = ColorFormat::RAW;
ColorFormat out_fmt = ColorFormat::BGR;
std::tie(prec, interp, in_fmt, in_layout, out_layout, ocv_channels, sizes) = GetParam();
std::tie(precisions, interp, in_fmt, in_layout, out_layout, ocv_channels, sizes) = GetParam();
Precision in_prec, out_prec;
std::tie(in_prec, out_prec) = precisions;
cv::Size in_size, out_size;
std::tie(in_size, out_size) = sizes;
int in_ocv_chan = -1, out_ocv_chan = -1;
@@ -1096,11 +1102,21 @@ TEST_P(PreprocTest, Performance)
double tolerance = Precision::U8 ? 1 : 0.015;
#endif
const int ocv_depth = prec == Precision::U8 ? CV_8U :
prec == Precision::FP32 ? CV_32F : -1;
const int in_ocv_type = CV_MAKETYPE(ocv_depth, in_ocv_chan);
const int out_ocv_type = CV_MAKETYPE(ocv_depth, out_ocv_chan);
initMatrixRandU(in_ocv_type, in_size, in_ocv_type, false);
auto precision_to_depth = [](Precision::ePrecision prec) -> int{
switch (prec)
{
case Precision::U8: return CV_8U;
case Precision::U16: return CV_16U;
case Precision::FP32: return CV_32F;
default:
throw std::logic_error("Unsupported configuration");
}
return -1;
};
const int in_ocv_type = CV_MAKETYPE(precision_to_depth(in_prec), in_ocv_chan);
const int out_ocv_type = CV_MAKETYPE(precision_to_depth(out_prec), out_ocv_chan);
initMatrixRandU(in_ocv_type, in_size, out_ocv_type, false);
cv::Mat out_mat(out_size, out_ocv_type);
@@ -1115,28 +1131,37 @@ TEST_P(PreprocTest, Performance)
cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255));
}
Blob::Ptr in_blob, out_blob;
switch (prec)
{
case Precision::U8:
if (in_fmt == ColorFormat::NV12) {
auto y_blob = img2Blob<Precision::U8>(in_mat1, Layout::NHWC);
auto uv_blob = img2Blob<Precision::U8>(in_mat2, Layout::NHWC);
in_blob = make_shared_blob<NV12Blob>(y_blob, uv_blob);
} else {
in_blob = img2Blob<Precision::U8>(in_mat1, in_layout);
auto create_blob = [](Precision::ePrecision prec, ColorFormat fmt, Layout layout, cv::Mat& m1, cv::util::optional<cv::Mat> m2 = {}){
Blob::Ptr blob;
switch (prec)
{
case Precision::U8:
if (fmt == ColorFormat::NV12) {
auto y_blob = img2Blob<Precision::U8>(m1, Layout::NHWC);
auto uv_blob = img2Blob<Precision::U8>(m2.value(), Layout::NHWC);
blob = make_shared_blob<NV12Blob>(y_blob, uv_blob);
} else {
blob = img2Blob<Precision::U8>(m1, layout);
}
break;
case Precision::U16:
blob = img2Blob<Precision::U16>(m1, layout);
break;
case Precision::FP32:
blob = img2Blob<Precision::FP32>(m1, layout);
break;
default:
throw std::logic_error("Unsupported configuration");
}
out_blob = img2Blob<Precision::U8>(out_mat, out_layout);
break;
return blob;
};
case Precision::FP32:
in_blob = img2Blob<Precision::FP32>(in_mat1, in_layout);
out_blob = img2Blob<Precision::FP32>(out_mat, out_layout);
break;
default:
FAIL() << "Unsupported configuration";
}
auto in_blob = create_blob(in_prec, in_fmt, in_layout, in_mat1, cv::util::make_optional(in_mat2));
auto out_blob = create_blob(out_prec, out_fmt, out_layout, out_mat);
PreProcessDataPtr preprocess = CreatePreprocDataHelper();
preprocess->setRoiBlob(in_blob);
@@ -1148,10 +1173,11 @@ TEST_P(PreprocTest, Performance)
// test once to warm-up cache
preprocess->execute(out_blob, info, false);
switch (prec)
switch (out_prec)
{
case Precision::U8: Blob2Img<Precision::U8> (out_blob, out_mat, out_layout); break;
case Precision::FP32: Blob2Img<Precision::FP32>(out_blob, out_mat, out_layout); break;
case Precision::U16: Blob2Img<Precision::FP32>(out_blob, out_mat, out_layout); break;
default: FAIL() << "Unsupported configuration";
}
@@ -1166,11 +1192,18 @@ TEST_P(PreprocTest, Performance)
auto cv_interp = interp == RESIZE_AREA ? cv::INTER_AREA : cv::INTER_LINEAR;
cv::resize(ocv_out_mat, ocv_out_mat, out_size, 0, 0, cv_interp);
if (in_prec != out_prec) {
cv::Mat ocv_converted;
ocv_out_mat.convertTo(ocv_converted, out_ocv_type);
ocv_out_mat = ocv_converted;
}
EXPECT_LE(cv::norm(ocv_out_mat, out_mat, cv::NORM_INF), tolerance);
#if PERF_TEST
// iterate testing, and print performance
const auto type_str = depthToString(ocv_depth);
const auto in_type_str = depthToString(precision_to_depth(in_prec));
const auto out_type_str = depthToString(precision_to_depth(out_prec));
const auto interp_str = interp == RESIZE_AREA ? "AREA"
: interp == RESIZE_BILINEAR ? "BILINEAR" : "?";
const auto in_layout_str = layoutToString(in_layout);
@@ -1178,8 +1211,9 @@ TEST_P(PreprocTest, Performance)
test_ms([&]() { preprocess->execute(out_blob, info, false); },
300,
"Preproc %s %s %d %s %dx%d %d %s %dx%d %s->%s",
type_str.c_str(),
"Preproc %s %s %s %d %s %dx%d %d %s %dx%d %s->%s",
in_type_str.c_str(),
out_type_str.c_str(),
interp_str,
in_ocv_chan,
in_layout_str.c_str(), in_size.width, in_size.height,

9
inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.hpp
View File

@@ -48,9 +48,16 @@ struct ColorConvertYUV420TestIE:
double>> // tolerance
{};
struct PrecisionConvertTestIE: public TestParams<std::tuple<cv::Size,
int, // input matrix depth
int, // output matrix depth
double>> // tolerance
{};
//------------------------------------------------------------------------------
using PreprocParams = std::tuple< InferenceEngine::Precision // input-output data type
using PreprocParams = std::tuple< std::pair<InferenceEngine::Precision // input data type
, InferenceEngine::Precision> // output data type
, InferenceEngine::ResizeAlgorithm // resize algorithm, if needed
, InferenceEngine::ColorFormat // input color format, if needed
, InferenceEngine::Layout // input tensor layout

43
inference-engine/tests_deprecated/fluid_preproc/cpu/fluid_tests_cpu.cpp
View File

@@ -323,8 +323,12 @@ static const auto PATCH_SIZES =
cv::Size(80, 160))); // 80 - person-attributes-recognition-crossroad-0031
// 64 - person-reidentification-retail-0079
static const auto U8toU8 = std::make_pair(IE::Precision::U8,IE::Precision::U8);
static const auto PRECISIONS = Values(U8toU8, std::make_pair(IE::Precision::FP32,IE::Precision::FP32));
INSTANTIATE_TEST_CASE_P(ReorderResize_Frame, PreprocTest,
Combine(Values(IE::Precision::U8, IE::Precision::FP32),
Combine(PRECISIONS,
Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet
Values(IE::ColorFormat::RAW),
Values(IE::Layout::NHWC),
@@ -333,7 +337,7 @@ INSTANTIATE_TEST_CASE_P(ReorderResize_Frame, PreprocTest,
FRAME_SIZES));
INSTANTIATE_TEST_CASE_P(Scale3ch_Frame, PreprocTest,
Combine(Values(IE::Precision::U8, IE::Precision::FP32),
Combine(PRECISIONS,
Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet
Values(IE::ColorFormat::RAW),
Values(IE::Layout::NHWC),
@@ -342,7 +346,7 @@ INSTANTIATE_TEST_CASE_P(Scale3ch_Frame, PreprocTest,
FRAME_SIZES));
INSTANTIATE_TEST_CASE_P(ReorderResize_Patch, PreprocTest,
Combine(Values(IE::Precision::U8, IE::Precision::FP32),
Combine(PRECISIONS,
Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet
Values(IE::ColorFormat::RAW),
Values(IE::Layout::NHWC),
@@ -351,7 +355,7 @@ INSTANTIATE_TEST_CASE_P(ReorderResize_Patch, PreprocTest,
PATCH_SIZES));
INSTANTIATE_TEST_CASE_P(Everything_Resize, PreprocTest,
Combine(Values(IE::Precision::U8, IE::Precision::FP32),
Combine(PRECISIONS,
Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA),
Values(IE::ColorFormat::RAW),
Values(IE::Layout::NHWC, IE::Layout::NCHW),
@@ -363,7 +367,7 @@ INSTANTIATE_TEST_CASE_P(Everything_Resize, PreprocTest,
Values(TEST_SIZES_PREPROC)));
INSTANTIATE_TEST_CASE_P(ColorFormats_3ch, PreprocTest,
Combine(Values(IE::Precision::U8, IE::Precision::FP32),
Combine(PRECISIONS,
Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA),
Values(IE::ColorFormat::RGB),
Values(IE::Layout::NHWC, IE::Layout::NCHW),
@@ -372,7 +376,7 @@ INSTANTIATE_TEST_CASE_P(ColorFormats_3ch, PreprocTest,
Values(TEST_SIZES_PREPROC)));
INSTANTIATE_TEST_CASE_P(ColorFormats_4ch, PreprocTest,
Combine(Values(IE::Precision::U8, IE::Precision::FP32),
Combine(PRECISIONS,
Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA),
Values(IE::ColorFormat::BGRX, IE::ColorFormat::RGBX),
Values(IE::Layout::NHWC),
@@ -381,10 +385,35 @@ INSTANTIATE_TEST_CASE_P(ColorFormats_4ch, PreprocTest,
Values(TEST_SIZES_PREPROC)));
INSTANTIATE_TEST_CASE_P(ColorFormat_NV12, PreprocTest,
Combine(Values(IE::Precision::U8),
Combine(Values(U8toU8),
Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA),
Values(IE::ColorFormat::NV12),
Values(IE::Layout::NCHW),
Values(IE::Layout::NHWC, IE::Layout::NCHW),
Values(std::make_pair(1, 3)),
Values(TEST_SIZES_PREPROC)));
INSTANTIATE_TEST_CASE_P(DISABLED_PlainPrecisionConversions, PreprocTest,
Combine(Values(std::make_pair(IE::Precision::U16,IE::Precision::FP32),
std::make_pair(IE::Precision::FP32,IE::Precision::U16)
),
Values(IE::ResizeAlgorithm::NO_RESIZE),
Values(IE::ColorFormat::RAW),
Values(IE::Layout::NHWC),
Values(IE::Layout::NHWC),
Values(std::make_pair(1, 1)),
Values(std::make_pair(cv::Size(640,480), cv::Size(640,480)))));
INSTANTIATE_TEST_CASE_P(PrecisionConversionsPipelines, PreprocTest,
Combine(Values(std::make_pair(IE::Precision::U16, IE::Precision::FP32),
std::make_pair(IE::Precision::FP32,IE::Precision::U8),
std::make_pair(IE::Precision::U8, IE::Precision::FP32)
),
Values(IE::ResizeAlgorithm::RESIZE_BILINEAR),
Values(IE::ColorFormat::RAW),
Values(IE::Layout::NHWC, IE::Layout::NCHW),
Values(IE::Layout::NHWC, IE::Layout::NCHW),
Values(std::make_pair(1, 1)/*, std::make_pair(3, 3)*/), //U16 Split and Merge are not there
Values(TEST_SIZES_PREPROC)));