IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[IE] Add Blob::createROI method (#882)

Browse Source 
* Add default implementation that throws exception.
* Implement `createROI` for `TBlob` and existing compound blobs.
* Use reference couting for TBlob memory buffer to prolong its life time for ROI blobs.
* Add private extension for ND ROI and use it as implementation detail for now:
  * Add `DimSlice` and `TensorSlice` structures for generic ND ROI support.
  * Add `make_roi_desc` function to create `TensorDesc` for ROI.
This commit is contained in:
Vladislav Vinogradov 2020-07-28 11:26:38 +03:00 committed by GitHub
parent a19a8645e8
commit 0b1ef99fd7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 329 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
inference-engine/include/ie_blob.h
View File

@ -29,6 +29,7 @@
#include "ie_precision.hpp"
namespace InferenceEngine {
/**
* @brief This class represents a universal container in the Inference Engine
*
@ -199,6 +200,17 @@ public:
*/
virtual LockedMemory<const void> cbuffer() const noexcept = 0;
/**
* @brief Creates a blob describing given ROI object based on the current blob with memory sharing.
*
* Note: default implementation throws "not implemented" exception.
*
* @param roi A ROI object inside of the current blob.
*
* @return A shared pointer to the newly created ROI blob.
*/
virtual Blob::Ptr createROI(const ROI& roi) const;
protected:
/**
* @brief The tensor descriptor of the given blob.
@ -437,8 +449,6 @@ public:
*/
virtual LockedMemory<void> wmap()noexcept = 0;
protected:
/**
* @brief Gets the allocator for allocator-based blobs.
@ -594,10 +604,18 @@ public:
* @brief Allocates or reallocates memory
*/
void allocate() noexcept override {
if (_handle != nullptr) {
getAllocator()->free(_handle);
const auto allocator = getAllocator();
const auto rawHandle = allocator->alloc(size() * sizeof(T));
if (rawHandle == nullptr) {
return;
}
_handle = getAllocator()->alloc(size() * sizeof(T));
_handle.reset(
rawHandle,
[allocator](void* rawHandle) {
allocator->free(rawHandle);
});
}
/**
@ -636,6 +654,10 @@ public:
return std::move(lockme<void>());
}
Blob::Ptr createROI(const ROI& roi) const override {
return Blob::Ptr(new TBlob<T>(*this, roi));
}
/**
* @brief Gets BlobIterator for the data.
*
@ -689,7 +711,7 @@ protected:
/**
* @brief A handle for the stored memory returned from _allocator.alloc().
*/
void* _handle = nullptr;
std::shared_ptr<void> _handle;
/**
* @brief Copies dimensions and data from the TBlob object.
@ -720,8 +742,8 @@ protected:
* @brief Frees handler and cleans up the stored data.
*/
virtual bool free() {
bool bCanRelease = getAllocator()->free(_handle);
_handle = nullptr;
bool bCanRelease = _handle != nullptr;
_handle.reset();
return bCanRelease;
}
@ -733,7 +755,7 @@ protected:
*/
template <class S>
LockedMemory<S> lockme() const {
return LockedMemory<S>(_allocator.get(), _handle, 0);
return LockedMemory<S>(_allocator.get(), getHandle(), 0);
}
/**
@ -754,7 +776,16 @@ protected:
* @brief Returns handle to the stored data.
*/
void* getHandle() const noexcept override {
return _handle;
return _handle.get();
}
TBlob(const TBlob& origBlob, const ROI& roi) :
MemoryBlob(make_roi_desc(origBlob.getTensorDesc(), roi, true)),
_allocator(origBlob._allocator) {
IE_ASSERT(origBlob._handle != nullptr)
<< "Original Blob must be allocated before ROI creation";
_handle = origBlob._handle;
}
};
@ -846,17 +877,6 @@ std::shared_ptr<T> make_shared_blob(Args&&... args) {
return std::make_shared<T>(std::forward<Args>(args)...);
}
/**
* @brief This structure describes ROI data.
*/
struct ROI {
size_t id; //!< ID of a ROI
size_t posX; //!< W upper left coordinate of ROI
size_t posY; //!< H upper left coordinate of ROI
size_t sizeX; //!< W size of ROI
size_t sizeY; //!< H size of ROI
};
/**
* @brief Creates a blob describing given ROI object based on the given blob with pre-allocated memory.
*

6
inference-engine/include/ie_compound_blob.h
View File

@ -117,6 +117,8 @@ public:
*/
virtual Blob::Ptr getBlob(size_t i) const noexcept;
Blob::Ptr createROI(const ROI& roi) const override;
protected:
/**
* @brief A default constructor
@ -219,6 +221,8 @@ public:
* @brief Returns a shared pointer to UV plane
*/
virtual const Blob::Ptr& uv() const noexcept;
Blob::Ptr createROI(const ROI& roi) const override;
};
/**
@ -342,5 +346,7 @@ public:
* @return constant reference to shared pointer object of V plane
*/
const Blob::Ptr& v() const noexcept;
Blob::Ptr createROI(const ROI& roi) const override;
};
} // namespace InferenceEngine

32
inference-engine/include/ie_layouts.h
View File

@ -318,4 +318,36 @@ private:
BlockingDesc blockingDesc;
};
/**
* @brief This structure describes ROI data for image-like tensors.
*/
struct ROI {
size_t id = 0; //!< ID of a ROI (offset over batch dimension)
size_t posX = 0; //!< W upper left coordinate of ROI
size_t posY = 0; //!< H upper left coordinate of ROI
size_t sizeX = 0; //!< W size of ROI
size_t sizeY = 0; //!< H size of ROI
ROI() = default;
ROI(size_t id, size_t posX, size_t posY, size_t sizeX, size_t sizeY) :
id(id), posX(posX), posY(posY), sizeX(sizeX), sizeY(sizeY) {
}
};
/**
* @brief Creates a TensorDesc object for ROI.
*
* @param origDesc original TensorDesc object.
* @param roi An image ROI object inside of the original object.
* @param useOrigMemDesc Flag to use original memory description (strides/offset).
* Should be set if the new TensorDesc describes shared memory.
*
* @return A newly created TensorDesc object representing ROI.
*/
INFERENCE_ENGINE_API_CPP(TensorDesc) make_roi_desc(
const TensorDesc& origDesc,
const ROI& roi,
bool useOrigMemDesc);
} // namespace InferenceEngine

1
inference-engine/src/inference_engine/CMakeLists.txt
View File

@ -20,6 +20,7 @@ set(IE_BASE_SOURCE_FILES
${CMAKE_CURRENT_SOURCE_DIR}/cnn_network_ngraph_impl.cpp
${CMAKE_CURRENT_SOURCE_DIR}/generic_ie.cpp
${CMAKE_CURRENT_SOURCE_DIR}/blob_factory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ie_blob_common.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ie_data.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ie_layouts.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ie_memcpy.cpp

55
inference-engine/src/inference_engine/ie_blob_common.cpp
View File

@ -2,61 +2,20 @@
// SPDX-License-Identifier: Apache-2.0
//
#include "ie_blob.h"
#include <memory>
#include <utility>
#include <vector>
#include "blob_factory.hpp"
#include "ie_blob.h"
#include "ie_compound_blob.h"
namespace InferenceEngine {
Blob::Ptr Blob::createROI(const ROI&) const {
THROW_IE_EXCEPTION << "[NOT_IMPLEMENTED] createROI is not implemented for current type of Blob";
}
Blob::Ptr make_shared_blob(const Blob::Ptr& inputBlob, const ROI& roi) {
// reject compound blobs
if (inputBlob->is<CompoundBlob>()) {
THROW_IE_EXCEPTION << "Compound blobs do not support ROI";
}
size_t blkDimsH = roi.sizeY;
size_t blkDimsW = roi.sizeX;
size_t blkDimsC = inputBlob->getTensorDesc().getDims()[1];
size_t blkOffset;
SizeVector blkOrder;
SizeVector blkDims;
if (roi.posX + roi.sizeX > inputBlob->getTensorDesc().getDims()[3] ||
roi.posY + roi.sizeY > inputBlob->getTensorDesc().getDims()[2]) {
THROW_IE_EXCEPTION << "passed ROI coordinates are inconsistent to input size";
}
Layout blobLayout = inputBlob->getTensorDesc().getLayout();
switch (blobLayout) {
case NCHW: {
blkOffset = inputBlob->getTensorDesc().getDims()[3] * roi.posY + roi.posX;
blkOrder = {0, 1, 2, 3};
blkDims = {1, blkDimsC, blkDimsH, blkDimsW}; // we use BlockingDesc for 1 cropped image only
} break;
case NHWC: {
blkOffset = blkDimsC * (inputBlob->getTensorDesc().getDims()[3] * roi.posY + roi.posX);
blkOrder = {0, 2, 3, 1};
blkDims = {1, blkDimsH, blkDimsW, blkDimsC}; // we use BlockingDesc for 1 cropped image only
} break;
default: {
THROW_IE_EXCEPTION << "ROI could not be cropped due to inconsistent input layout: " << blobLayout;
}
}
// the strides are the same because ROI blob uses the same memory buffer as original input blob.
SizeVector blkStrides(inputBlob->getTensorDesc().getBlockingDesc().getStrides());
SizeVector blkDimsOffsets = {0, 0, 0, 0}; // no offset per dims by default
BlockingDesc blkDesc(blkDims, blkOrder, blkOffset, blkDimsOffsets, blkStrides);
TensorDesc tDesc(inputBlob->getTensorDesc().getPrecision(), {1, blkDimsC, blkDimsH, blkDimsW}, blkDesc);
tDesc.setLayout(blobLayout);
return make_blob_with_precision(tDesc, inputBlob->buffer());
return inputBlob->createROI(roi);
}
} // namespace InferenceEngine

41
inference-engine/src/inference_engine/ie_compound_blob.cpp
View File

@ -15,7 +15,9 @@
#include <vector>
namespace InferenceEngine {
namespace {
void verifyNV12BlobInput(const Blob::Ptr& y, const Blob::Ptr& uv) {
// Y and UV must be valid pointers
if (y == nullptr || uv == nullptr) {
@ -189,6 +191,7 @@ void verifyI420BlobInput(const Blob::Ptr& y, const Blob::Ptr& u, const Blob::Ptr
<< yDims[3] << "(Y plane) and " << vDims[3] << "(V plane)";
}
}
} // anonymous namespace
CompoundBlob::CompoundBlob(): Blob(TensorDesc(Precision::UNSPECIFIED, {}, Layout::ANY)) {}
@ -272,6 +275,17 @@ Blob::Ptr CompoundBlob::getBlob(size_t i) const noexcept {
return _blobs[i];
}
Blob::Ptr CompoundBlob::createROI(const ROI& roi) const {
std::vector<Blob::Ptr> roiBlobs;
roiBlobs.reserve(_blobs.size());
for (const auto& blob : _blobs) {
roiBlobs.push_back(blob->createROI(roi));
}
return std::make_shared<CompoundBlob>(std::move(roiBlobs));
}
const std::shared_ptr<IAllocator>& CompoundBlob::getAllocator() const noexcept {
static std::shared_ptr<IAllocator> _allocator = nullptr;
return _allocator;
@ -319,6 +333,19 @@ const Blob::Ptr& NV12Blob::uv() const noexcept {
return _blobs[1];
}
Blob::Ptr NV12Blob::createROI(const ROI& roi) const {
auto yROI = roi;
yROI.sizeX += yROI.sizeX % 2;
yROI.sizeY += yROI.sizeY % 2;
const auto uvROI = ROI(yROI.id, yROI.posX / 2, yROI.posY / 2, yROI.sizeX / 2, yROI.sizeY / 2);
const auto yRoiBlob = y()->createROI(yROI);
const auto uvRoiBlob = uv()->createROI(uvROI);
return std::make_shared<NV12Blob>(yRoiBlob, uvRoiBlob);
}
I420Blob::I420Blob(const Blob::Ptr& y, const Blob::Ptr& u, const Blob::Ptr& v) {
// verify data is correct
verifyI420BlobInput(y, u, v);
@ -371,4 +398,18 @@ const Blob::Ptr& I420Blob::v() const noexcept {
return _blobs[2];
}
Blob::Ptr I420Blob::createROI(const ROI& roi) const {
auto yROI = roi;
yROI.sizeX += yROI.sizeX % 2;
yROI.sizeY += yROI.sizeY % 2;
const auto uvROI = ROI(yROI.id, yROI.posX / 2, yROI.posY / 2, yROI.sizeX / 2, yROI.sizeY / 2);
const auto yRoiBlob = y()->createROI(yROI);
const auto uRoiBlob = u()->createROI(uvROI);
const auto vRoiBlob = v()->createROI(uvROI);
return std::make_shared<I420Blob>(yRoiBlob, uRoiBlob, vRoiBlob);
}
} // namespace InferenceEngine

114
inference-engine/src/inference_engine/ie_layouts.cpp
View File

@ -368,3 +368,117 @@ bool BlockingDesc::operator==(const BlockingDesc& rhs) const {
bool BlockingDesc::operator!=(const BlockingDesc& rhs) const {
return !(*this == rhs);
}
namespace {
struct DimSlice {
size_t startInd = 0;
size_t size = 0;
DimSlice() = default;
DimSlice(size_t startInd, size_t size) :
startInd(startInd), size(size) {
}
};
using TensorSlice = std::vector<DimSlice>;
void checkROI(
const TensorDesc& origDesc,
const TensorSlice& roi) {
const auto numDims = origDesc.getDims().size();
if (roi.size() != numDims) {
THROW_IE_EXCEPTION
<< "ROI num dims " << roi.size() <<
" differs from original num dims " << numDims;
}
// TensorDesc stores dimensions in standard layout, as well as roi vector
for (size_t dimInd = 0; dimInd < numDims; ++dimInd) {
const auto fullSize = origDesc.getDims()[dimInd];
const auto& roiSlice = roi[dimInd];
const auto endInd = roiSlice.startInd + roiSlice.size;
if (endInd > fullSize) {
THROW_IE_EXCEPTION
<< "ROI [" << roiSlice.startInd << ", " << endInd << ")"
<< " is out of range " << fullSize
<< " for dimension " << dimInd;
}
}
}
TensorDesc make_roi_desc(
const TensorDesc& origDesc,
const TensorSlice& roi,
bool useOrigMemDesc) {
const auto numDims = origDesc.getDims().size();
checkROI(origDesc, roi);
const auto origPrecision = origDesc.getPrecision();
const auto& origBlkDesc = origDesc.getBlockingDesc();
const auto& origBlkStrides = origBlkDesc.getStrides();
const auto& origBlkOrder = origBlkDesc.getOrder();
SizeVector roiDims(numDims);
SizeVector roiBlkDims(numDims);
SizeVector roiBlkDimOffsets = origBlkDesc.getOffsetPaddingToData();
size_t roiBlkOffset = origBlkDesc.getOffsetPadding();
IE_ASSERT(origBlkStrides.size() == numDims);
IE_ASSERT(origBlkOrder.size() == numDims);
IE_ASSERT(roiBlkDimOffsets.size() == numDims);
// BlockingDesc stores dimensions in memory order, so we need to use origOrder array.
// Offsets in `roi` relates to `origDesc` dimensions, while offsets in `BlockingDesc` relates to top parent tensor dimensions.
for (size_t memInd = 0; memInd < numDims; ++memInd) {
const auto dimInd = origBlkOrder[memInd];
const auto& roiSlice = roi[dimInd];
roiDims[dimInd] = roiSlice.size;
roiBlkDims[memInd] = roiSlice.size;
roiBlkDimOffsets[memInd] += roiSlice.startInd;
roiBlkOffset += roiSlice.startInd * origBlkStrides[memInd];
}
const auto roiBlkDesc =
useOrigMemDesc ?
BlockingDesc(roiBlkDims, origBlkOrder, roiBlkOffset, roiBlkDimOffsets, origBlkStrides) :
BlockingDesc(roiBlkDims, origBlkOrder);
const auto roiDesc = TensorDesc(origPrecision, roiDims, roiBlkDesc);
return roiDesc;
}
TensorSlice make_roi_slice(
const TensorDesc& origDesc,
const ROI& roi) {
const auto layout = origDesc.getLayout();
if (layout != Layout::NCHW && layout != Layout::NHWC) {
THROW_IE_EXCEPTION
<< "Unsupported layout " << layout;
}
TensorSlice roiSlice(4);
roiSlice[0] = DimSlice {roi.id, 1}; // N
roiSlice[1] = DimSlice {0, origDesc.getDims()[1]}; // C
roiSlice[2] = DimSlice {roi.posY, roi.sizeY}; // H
roiSlice[3] = DimSlice {roi.posX, roi.sizeX}; // W
return roiSlice;
}
} // namespace
TensorDesc InferenceEngine::make_roi_desc(
const TensorDesc& origDesc,
const ROI& roi,
bool useOrigMemDesc) {
return make_roi_desc(origDesc, make_roi_slice(origDesc, roi), useOrigMemDesc);
}

50
inference-engine/tests/unit/inference_engine/ie_blob_test.cpp
View File

@ -395,3 +395,53 @@ TEST_F(BlobTests, makeRoiBlobWrongSize) {
InferenceEngine::ROI roi = {0, 1, 1, 4, 4}; // cropped picture with: id = 0, (x,y) = (1,1), sizeX (W) = 4, sizeY (H) = 4
ASSERT_THROW(make_shared_blob(blob, roi), InferenceEngine::details::InferenceEngineException);
}
TEST_F(BlobTests, readRoiBlob) {
// Create original Blob
const auto origDesc =
InferenceEngine::TensorDesc(
InferenceEngine::Precision::I32,
{1, 3, 4, 8},
InferenceEngine::NCHW);
const auto origBlob =
InferenceEngine::make_shared_blob<int32_t>(origDesc);
origBlob->allocate();
// Fill the original Blob
{
auto origMemory = origBlob->wmap();
const auto origPtr = origMemory.as<int32_t*>();
ASSERT_NE(nullptr, origPtr);
for (size_t i = 0; i < origBlob->size(); ++i) {
origPtr[i] = i;
}
}
// Create ROI Blob
const auto roi = InferenceEngine::ROI(0, 4, 2, 4, 2);
const auto roiBlob = InferenceEngine::as<InferenceEngine::MemoryBlob>(origBlob->createROI(roi));
ASSERT_NE(nullptr, roiBlob);
// Read ROI Blob
{
const auto roiOffset = roiBlob->getTensorDesc().getBlockingDesc().getOffsetPadding();
auto roiMemory = roiBlob->rmap();
auto roiPtr = roiMemory.as<const int32_t*>();
ASSERT_NE(nullptr, roiPtr);
// Blob::rmap returns pointer to the original blob start, we have to add ROI offset manually.
roiPtr += roiOffset;
for (size_t i = 0; i < roiBlob->size(); ++i) {
ASSERT_EQ(roiPtr[i], i + roiOffset);
}
}
}

48
inference-engine/tests_deprecated/functional/shared_tests/io_blob_tests/cropResize_tests.hpp
View File

@ -398,15 +398,27 @@ TEST_P(RandomROITest, PreprocRandomROITest)
if (_colorFormat == NV12)
{
roi.sizeX += roi.sizeX % 2;
roi.sizeY += roi.sizeY % 2;
if (i % 2)
{
// New way to create NV12 ROI
auto roiUV = roi/2;
auto nv12Blob = make_shared_blob<NV12Blob>(yBlob, uvBlob);
cropBlob = nv12Blob->createROI(roi);
}
else
{
// Old way to create NV12 ROI
auto cropYBlob = make_shared_blob(yBlob, roi);
auto cropUvBlob = make_shared_blob(uvBlob, roiUV);
roi.sizeX += roi.sizeX % 2;
roi.sizeY += roi.sizeY % 2;
cropBlob = make_shared_blob<NV12Blob>(cropYBlob, cropUvBlob);
auto roiUV = roi/2;
auto cropYBlob = make_shared_blob(yBlob, roi);
auto cropUvBlob = make_shared_blob(uvBlob, roiUV);
cropBlob = make_shared_blob<NV12Blob>(cropYBlob, cropUvBlob);
}
}
else
{
@ -1110,13 +1122,27 @@ TEST_P(NV12ColorConvertTest, NV12Test) {
cv::resize(refImg, refImg, cv::Size(_netDims[3], _netDims[2]), 0, 0, cv_interpolation);
auto refBlob = img2Blob<Precision::FP32>(refImg, Layout::NCHW);
// Note: Y and UV blobs for original data must always be "alive" until the end of the execution:
// ROI blobs do not own the data
auto yBlob = img2Blob<Precision::U8>(yPlane, NHWC);
auto uvBlob = img2Blob<Precision::U8>(uvPlane, NHWC);
auto croppedYBlob = make_shared_blob(yBlob, yRoi);
auto croppedUvBlob = make_shared_blob(uvBlob, uvRoi);
auto inputBlob = make_shared_blob<NV12Blob>(croppedYBlob, croppedUvBlob);
Blob::Ptr inputBlob;
if (i % 2)
{
// New way to create NV12 ROI
auto nv12Blob = make_shared_blob<NV12Blob>(yBlob, uvBlob);
inputBlob = nv12Blob->createROI(yRoi);
}
else
{
// Old way to create NV12 ROI
// Note: Y and UV blobs for original data must always be "alive" until the end of the execution:
// ROI blobs do not own the data
auto croppedYBlob = make_shared_blob(yBlob, yRoi);
auto croppedUvBlob = make_shared_blob(uvBlob, uvRoi);
inputBlob = make_shared_blob<NV12Blob>(croppedYBlob, croppedUvBlob);
}
req.SetBlob(net.getInputsInfo().begin()->first, inputBlob);