IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Extended preprocessing for CPU (#5750)

Browse Source
This commit is contained in:
Yury Gaydaychuk 2021-06-01 17:03:24 +03:00 committed by GitHub
parent 6bf7ed83ab
commit 1264376173
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 72 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/template_plugin/src/transformations/preprocessing/preprocessing.hpp
View File

@ -20,10 +20,10 @@ class AddPreprocessing;
* @brief Converts the following preprocessing information to ngraph operations:
* - InferenceEngine::PreProcessInfo->PreProcessChannel::meanData -> Subtract
* - InferenceEngine::PreProcessInfo->PreProcessChannel::meanValue -> Subtract
* - InferenceEngine::PreProcessInfo->PreProcessChannel::stdScale -> Multiply
* - InferenceEngine::PreProcessInfo->PreProcessChannel::stdScale -> Divide
*
* The order of operations is the following:
* (x - mean) * stdScale
* (x - mean) / stdScale
*/
class ngraph::pass::AddPreprocessing : public ngraph::pass::FunctionPass {
const InferenceEngine::InputsDataMap& m_inputInfoMap;

6
docs/template_plugin/src/transformations/preprocessing/std_scale.cpp
View File

@ -31,10 +31,10 @@ ngraph::pass::AddStdScale::AddStdScale(const ScaleMap& inputInfoMap) {
NGRAPH_CHECK(scale_const->get_element_type() == ngraph::element::f32, "Scale for ", param->get_friendly_name(), " must have f32 type");
auto copy_param = param->clone_with_new_inputs({});
auto mul = std::make_shared<ngraph::opset3::Multiply>(copy_param, it->second);
auto div = std::make_shared<ngraph::opset3::Divide>(copy_param, it->second);
ngraph::replace_node(param, mul);
mul->set_argument(0, param);
ngraph::replace_node(param, div);
div->set_argument(0, param);
// Return true as the root node was changed
return true;

12
docs/template_plugin/tests/functional/transformations/preprocessing.cpp
View File

@ -43,8 +43,8 @@
// auto data = std::make_shared<opset5::Parameter>(element::f32, data_shape);
// auto scales = opset5::Constant::create(element::f32, scale_shape,
// std::vector<float>(shape_size(scale_shape), 2.0f));
// auto mul = std::make_shared<opset5::Multiply>(data, scales);
// auto relu = std::make_shared<opset5::Relu>(mul);
// auto div = std::make_shared<opset5::Divide>(data, scales);
// auto relu = std::make_shared<opset5::Relu>(div);
// f_ref = std::make_shared<Function>(NodeVector{relu}, ParameterVector{data});
// }
@ -137,8 +137,8 @@
// auto scaleValues = opset5::Constant::create(element::f32, scale_shape,
// std::vector<float>(shape_size(scale_shape), 2.0f));
// auto sub = std::make_shared<opset5::Subtract>(data, meanValues);
// auto mul = std::make_shared<opset5::Multiply>(sub, scaleValues);
// auto relu = std::make_shared<opset5::Relu>(mul);
// auto div = std::make_shared<opset5::Divide>(sub, scaleValues);
// auto relu = std::make_shared<opset5::Relu>(div);
// f_ref = std::make_shared<Function>(NodeVector{relu}, ParameterVector{data});
// }
@ -173,8 +173,8 @@
// auto scaleValues = opset5::Constant::create(element::f32, scale_shape,
// std::vector<float>(shape_size(scale_shape), 2.0f));
// auto sub = std::make_shared<opset5::Subtract>(data, meanValues);
// auto mul = std::make_shared<opset5::Multiply>(sub, meanValues);
// auto relu = std::make_shared<opset5::Relu>(mul);
// auto div = std::make_shared<opset5::Divide>(sub, meanValues);
// auto relu = std::make_shared<opset5::Relu>(div);
// f_ref = std::make_shared<Function>(NodeVector{relu}, ParameterVector{data});
// }

9
inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp
View File

@ -311,7 +311,7 @@ void MKLDNNGraph::Replicate(const CNNNetwork &network, const MKLDNNExtensionMana
}
InputInfo::Ptr ii = inputsInfo[input.first];
if (ii && ii->getPreProcess().getNumberOfChannels()) {
_meanImages[input.first].Load(outDims, ii);
_normalizePreprocMap[input.first].Load(outDims, ii);
}
}
}
@ -362,7 +362,7 @@ void MKLDNNGraph::InitDescriptors() {
OV_ITT_SCOPE_CHAIN(FIRST_INFERENCE, taskChain, MKLDNNPlugin::itt::domains::MKLDNN_LT, "InitDescriptors", "Prepare");
for (auto &node : graphNodes) {
if (node->getType() == Input && _meanImages.find(node->getName()) != _meanImages.end()) {
if (node->getType() == Input && _normalizePreprocMap.find(node->getName()) != _normalizePreprocMap.end()) {
auto *inputNode = dynamic_cast<MKLDNNInputNode *>(node.get());
if (inputNode)
inputNode->withMeanImage();
@ -726,9 +726,10 @@ void MKLDNNGraph::PushInputData(const std::string& name, const InferenceEngine::
}
// todo: make sure 'name' exists in this map...
if (_meanImages.find(name) != _meanImages.end()) {
if (_normalizePreprocMap.find(name) != _normalizePreprocMap.end()) {
if (in->getTensorDesc().getPrecision() == InferenceEngine::Precision::FP32) {
_meanImages[name].Subtract(outDims, reinterpret_cast<float *>(inter_data_ptr), in->getTensorDesc().getLayout());
_normalizePreprocMap[name].NormalizeImage(outDims, reinterpret_cast<float *>(inter_data_ptr),
in->getTensorDesc().getLayout());
} else {
IE_THROW() << "Mean image of type " << in->getTensorDesc().getPrecision().name() << " is unsupported";
}

8
inference-engine/src/mkldnn_plugin/mkldnn_graph.h
View File

@ -7,7 +7,7 @@
#include "cpp/ie_cnn_network.h"
#include "config.h"
#include "mkldnn_memory.h"
#include "mean_image.h"
#include "normalize_preprocess.h"
#include "mkldnn_node.h"
#include "mkldnn_edge.h"
#include <map>
@ -51,7 +51,7 @@ public:
MKLDNNWeightsSharing::Ptr &w_cache);
bool hasMeanImageFor(const std::string& name) {
return _meanImages.find(name) != _meanImages.end();
return _normalizePreprocMap.find(name) != _normalizePreprocMap.end();
}
void PushInputData(const std::string& name, const InferenceEngine::Blob::Ptr &in);
@ -176,7 +176,7 @@ protected:
outputNodesMap.clear();
graphNodes.clear();
graphEdges.clear();
_meanImages.clear();
_normalizePreprocMap.clear();
}
Status status { NotReady };
Config config;
@ -194,7 +194,7 @@ protected:
std::vector<MKLDNNNodePtr> graphNodes;
std::vector<MKLDNNEdgePtr> graphEdges;
std::map<std::string, MeanImage> _meanImages;
std::map<std::string, NormalizePreprocess> _normalizePreprocMap;
std::string _name;
bool isQuantizedFlag = false;

17
inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp
View File

@ -236,12 +236,25 @@ InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std::
_inputs[name] = make_blob_with_precision(desc);
_inputs[name]->allocate();
if (blobs[name]->getTensorDesc() == desc &&
graph->_meanImages.find(name) == graph->_meanImages.end() && !graph->getProperty().batchLimit) {
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = _inputs[name]->buffer();
}
}
data = _inputs[name];
checkBlob(data, name, true);
// check if preprocess required, but still wasn't set
auto preProcessedInput = std::find_if(std::begin(_networkInputs), std::end(_networkInputs),
[&](const std::pair<std::string, InferenceEngine::InputInfo::Ptr>& pair)
{return pair.first == name;});
if (preProcessedInput!= std::end(_networkInputs)) {
auto preProcess = preProcessedInput->second->getPreProcess();
if (preProcess.getColorFormat() != InferenceEngine::ColorFormat::RAW ||
preProcess.getResizeAlgorithm() != InferenceEngine::ResizeAlgorithm::NO_RESIZE) {
_preProcData.emplace(name, InferenceEngine::CreatePreprocDataHelper());
_preProcData[name]->isApplicable(data, _inputs[name]);
_preProcData[name]->setRoiBlob(data);
}
}
}
if (graph->hasOutputWithName(name)) {
@ -359,7 +372,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In
IE_THROW() << "MKLDNN graph doesn't contain input node with name: " << name;
if (data->getTensorDesc() == blobs.at(name)->getTensorDesc() &&
graph->_meanImages.find(name) == graph->_meanImages.end() && !graph->getProperty().batchLimit) {
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) {
externalPtr.erase(name);

24
inference-engine/src/mkldnn_plugin/mean_image.cpp → inference-engine/src/mkldnn_plugin/normalize_preprocess.cpp
View File

@ -2,17 +2,17 @@
// SPDX-License-Identifier: Apache-2.0
//
#include "mean_image.h"
#include "normalize_preprocess.h"
#include "ie_parallel.hpp"
#include "nodes/common/cpu_memcpy.h"
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
MeanImage::MeanImage() : meanBuffer(nullptr) {
NormalizePreprocess::NormalizePreprocess() : meanBuffer(nullptr) {
}
void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) {
void NormalizePreprocess::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) {
PreProcessInfo &pp = inputInfo->getPreProcess();
size_t inChannels = pp.getNumberOfChannels();
if (inChannels == 0) {
@ -26,11 +26,16 @@ void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) {
switch (pp.getMeanVariant()) {
case MEAN_VALUE: {
// mean image common value per channel (1x1xC)
// mean and standard deviation image common value per channel (1x1xC)
meanValues.resize(inChannels);
stdScales.resize(inChannels);
for (unsigned channel = 0; channel < inChannels; channel++) {
if (pp[channel]->stdScale == 0) {
IE_THROW() << "Preprocessing error: stdScale cannot be equal zero";
}
meanValues[channel] = pp[channel]->meanValue;
stdScales[channel] = pp[channel]->stdScale;
}
}
break;
@ -71,7 +76,7 @@ void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) {
}
}
void MeanImage::Subtract(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout) {
void NormalizePreprocess::NormalizeImage(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout) {
IE_ASSERT(input != nullptr);
if (inputDims.ndims() != 4) {
@ -91,19 +96,24 @@ void MeanImage::Subtract(const MKLDNNDims &inputDims, float *input, InferenceEng
parallel_for2d(MB, srcSize, [&](int mb, int i) {
input[srcSize * mb + i] -= meanBufferValues[i];
});
} else if (!meanValues.empty()) {
} else if (!meanValues.empty() && !stdScales.empty()) {
int C = inputDims[1];
srcSize /= inputDims[1];
if (layout == NCHW) {
parallel_for3d(MB, C, srcSize, [&](int mb, int c, int i) {
input[mb * C * srcSize + c * srcSize + i] -= meanValues[c];
input[mb * C * srcSize + c * srcSize + i] /= stdScales[c];
});
} else if (layout == NHWC) {
parallel_for2d(MB, srcSize, [&](int mb, int i) {
for (int c = 0; c < C; c++)
for (int c = 0; c < C; c++) {
input[mb * srcSize * C + i * C + c] -= meanValues[c];
input[mb * srcSize * C + i * C + c] /= stdScales[c];
}
});
}
} else {
IE_THROW() << "Preprocessing error: meanValues and stdScales arrays are inconsistent.";
}
}

19
inference-engine/src/mkldnn_plugin/mean_image.h → inference-engine/src/mkldnn_plugin/normalize_preprocess.h
View File

@ -13,16 +13,16 @@
namespace MKLDNNPlugin {
class MeanImage {
class NormalizePreprocess {
public:
MeanImage();
NormalizePreprocess();
public:
void Load(const MKLDNNDims& inputDims, InferenceEngine::InputInfo::Ptr inputInfo);
void Subtract(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout);
void NormalizeImage(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout);
template<typename T, typename std::enable_if<std::is_integral<T>::value>::type* = nullptr>
void Subtract(const MKLDNNDims &inputDims, T *input, InferenceEngine::Layout layout) {
void NormalizeImage(const MKLDNNDims &inputDims, T *input, InferenceEngine::Layout layout) {
IE_ASSERT(input != nullptr);
if (inputDims.ndims() != 4) {
@ -46,10 +46,15 @@ public:
if (buf > (std::numeric_limits<T>::max)()) buf = (std::numeric_limits<T>::max)();
input[srcSize * mb + i] = buf;
});
} else if (!meanValues.empty()) {
} else if (!meanValues.empty() && !stdScales.empty()) {
int C = inputDims[1];
srcSize /= inputDims[1];
for (int c = 0; c < C; c++) {
if (stdScales[c] != 1)
IE_THROW() << "Preprocessing error: fractional normalization is not supported for integer data. ";
}
if (layout == InferenceEngine::NCHW) {
InferenceEngine::parallel_for3d(MB, C, srcSize, [&](int mb, int c, int i) {
int buf = input[srcSize * mb * C + c * srcSize + i];
@ -69,12 +74,16 @@ public:
}
});
}
} else {
IE_THROW() << "Preprocessing error: meanValues and stdScales arrays are inconsistent.";
}
}
private:
std::vector<float> meanValues;
std::vector<float> stdScales;
InferenceEngine::TBlob<float>::Ptr meanBuffer;
};

5
inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp
View File

@ -24,11 +24,6 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*(RangeAddSubgraphTest).*Start=1.2.*Stop=(5.2|-5.2).*Step=(0.1|-0.1).*netPRC=FP16.*)",
R"(.*(RangeNumpyAddSubgraphTest).*netPRC=FP16.*)",
// TODO: Issue: 43793
R"(.*(PreprocessTest).*(SetScalePreProcessSetBlob).*)",
R"(.*(PreprocessTest).*(SetScalePreProcessGetBlob).*)",
R"(.*(PreprocessTest).*(SetMeanValuePreProcessSetBlob).*)",
R"(.*(PreprocessTest).*(SetMeanImagePreProcessSetBlob).*)",
R"(.*(PreprocessTest).*(ReverseInputChannelsPreProcessGetBlob).*)",
R"(.*PreprocessDynamicallyInSetBlobTest.*iPRC=0.*_iLT=1.*)",
R"(.*PreprocessDynamicallyInSetBlobTest.*oPRC=0.*_oLT=1.*)",
// TODO: Issue: 34348

3
inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp
View File

@ -37,6 +37,9 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*CTCGreedyDecoderSeqLen.*?\(1.1.1\).*)",
// TODO: Issue 51804
".*PreprocessConversionTest.*oPRC=U8.*",
// TODO: Issue: 56556
R"(.*(PreprocessTest).*(SetScalePreProcessSetBlob).*)",
R"(.*(PreprocessTest).*(SetScalePreProcessGetBlob).*)",
// TODO: Issue 54163
R"(.*ActivationLayerTest.*SoftPlus.*)",
// TODO: Issue 54722

9
inference-engine/tests/functional/plugin/shared/include/behavior/set_preprocess.hpp
View File

@ -323,11 +323,10 @@ TEST_P(PreprocessTest, SetMeanValuePreProcessSetBlob) {
const auto* outData = outMem.as<const float*>();
ASSERT_EQ(inBlob->size(), outBlob->size());
for (size_t i = 0; i < inBlob->size(); i++)
ASSERT_EQ(inData[i]+5, outData[i]);
ASSERT_EQ(inData[i] + 5, outData[i]);
}
}
TEST_P(PreprocessTest, ReverseInputChannelsPreProcessGetBlob) {
// Skip test according to plugin specific disabledTestPatterns() (if any)
SKIP_IF_CURRENT_TEST_IS_DISABLED()
@ -391,7 +390,6 @@ TEST_P(PreprocessTest, ReverseInputChannelsPreProcessGetBlob) {
}
}
TEST_P(PreprocessTest, ReverseInputChannelsPreProcessSetBlob) {
// Skip test according to plugin specific disabledTestPatterns() (if any)
SKIP_IF_CURRENT_TEST_IS_DISABLED()
@ -515,12 +513,11 @@ TEST_P(PreprocessTest, SetScalePreProcessGetBlob) {
const auto* outData = outMem.as<const float*>();
ASSERT_EQ(inBlob->size(), outBlob->size());
for (size_t i = 0; i < inBlob->size(); i++) {
ASSERT_EQ(inData[i]*2, outData[i]);
ASSERT_EQ(inData[i] / 2, outData[i]);
}
}
}
TEST_P(PreprocessTest, SetScalePreProcessSetBlob) {
// Skip test according to plugin specific disabledTestPatterns() (if any)
SKIP_IF_CURRENT_TEST_IS_DISABLED()
@ -581,7 +578,7 @@ TEST_P(PreprocessTest, SetScalePreProcessSetBlob) {
const auto* outData = outMem.as<const float*>();
ASSERT_EQ(inBlob->size(), outBlob->size());
for (size_t i = 0; i < inBlob->size(); i++)
ASSERT_EQ(inData[i]*2, outData[i]);
ASSERT_EQ(inData[i] / 2, outData[i]);
}
}