[IE] Add Blob::createROI method (#882)

* Add default implementation that throws exception.
* Implement `createROI` for `TBlob` and existing compound blobs.
* Use reference couting for TBlob memory buffer to prolong its life time for ROI blobs.
* Add private extension for ND ROI and use it as implementation detail for now:
  * Add `DimSlice` and `TensorSlice` structures for generic ND ROI support.
  * Add `make_roi_desc` function to create `TensorDesc` for ROI.

inference-engine/include/ie_blob.h

@@ -29,6 +29,7 @@
 #include "ie_precision.hpp"
 
 namespace InferenceEngine {
+
 /**
  * @brief This class represents a universal container in the Inference Engine
  *
@@ -199,6 +200,17 @@ public:
      */
     virtual LockedMemory<const void> cbuffer() const noexcept = 0;
 
+    /**
+     * @brief Creates a blob describing given ROI object based on the current blob with memory sharing.
+     *
+     * Note: default implementation throws "not implemented" exception.
+     *
+     * @param roi A ROI object inside of the current blob.
+     *
+     * @return A shared pointer to the newly created ROI blob.
+     */
+    virtual Blob::Ptr createROI(const ROI& roi) const;
+
 protected:
     /**
      * @brief The tensor descriptor of the given blob.
@@ -437,8 +449,6 @@ public:
      */
     virtual LockedMemory<void> wmap()noexcept = 0;
 
-
-
 protected:
     /**
      * @brief Gets the allocator for allocator-based blobs.
@@ -594,10 +604,18 @@ public:
      * @brief Allocates or reallocates memory
      */
     void allocate() noexcept override {
-        if (_handle != nullptr) {
+        const auto allocator = getAllocator();
-            getAllocator()->free(_handle);
+        const auto rawHandle = allocator->alloc(size() * sizeof(T));
+
+        if (rawHandle == nullptr) {
+            return;
         }
-        _handle = getAllocator()->alloc(size() * sizeof(T));
+
+        _handle.reset(
+            rawHandle,
+            [allocator](void* rawHandle) {
+                allocator->free(rawHandle);
+            });
     }
 
     /**
@@ -636,6 +654,10 @@ public:
         return std::move(lockme<void>());
     }
 
+    Blob::Ptr createROI(const ROI& roi) const override {
+        return Blob::Ptr(new TBlob<T>(*this, roi));
+    }
+
     /**
      * @brief Gets BlobIterator for the data.
      *
@@ -689,7 +711,7 @@ protected:
     /**
      * @brief A handle for the stored memory returned from _allocator.alloc().
      */
-    void* _handle = nullptr;
+    std::shared_ptr<void> _handle;
 
     /**
      * @brief Copies dimensions and data from the TBlob object.
@@ -720,8 +742,8 @@ protected:
      * @brief Frees handler and cleans up the stored data.
      */
     virtual bool free() {
-        bool bCanRelease = getAllocator()->free(_handle);
+        bool bCanRelease = _handle != nullptr;
-        _handle = nullptr;
+        _handle.reset();
         return bCanRelease;
     }
 
@@ -733,7 +755,7 @@ protected:
      */
     template <class S>
     LockedMemory<S> lockme() const {
-        return LockedMemory<S>(_allocator.get(), _handle, 0);
+        return LockedMemory<S>(_allocator.get(), getHandle(), 0);
     }
 
     /**
@@ -754,7 +776,16 @@ protected:
      * @brief Returns handle to the stored data.
      */
     void* getHandle() const noexcept override {
-        return _handle;
+        return _handle.get();
+    }
+
+    TBlob(const TBlob& origBlob, const ROI& roi) :
+            MemoryBlob(make_roi_desc(origBlob.getTensorDesc(), roi, true)),
+            _allocator(origBlob._allocator) {
+        IE_ASSERT(origBlob._handle != nullptr)
+            << "Original Blob must be allocated before ROI creation";
+
+        _handle = origBlob._handle;
     }
 };
 
@@ -846,17 +877,6 @@ std::shared_ptr<T> make_shared_blob(Args&&... args) {
     return std::make_shared<T>(std::forward<Args>(args)...);
 }
 
-/**
- * @brief This structure describes ROI data.
- */
-struct ROI {
-    size_t id;  //!< ID of a ROI
-    size_t posX;  //!< W upper left coordinate of ROI
-    size_t posY;  //!< H upper left coordinate of ROI
-    size_t sizeX;  //!< W size of ROI
-    size_t sizeY;  //!< H size of ROI
-};
-
 /**
  * @brief Creates a blob describing given ROI object based on the given blob with pre-allocated memory.
  *

inference-engine/include/ie_compound_blob.h

@@ -117,6 +117,8 @@ public:
      */
     virtual Blob::Ptr getBlob(size_t i) const noexcept;
 
+    Blob::Ptr createROI(const ROI& roi) const override;
+
 protected:
     /**
      * @brief A default constructor
@@ -219,6 +221,8 @@ public:
      * @brief Returns a shared pointer to UV plane
      */
     virtual const Blob::Ptr& uv() const noexcept;
+
+    Blob::Ptr createROI(const ROI& roi) const override;
 };
 
 /**
@@ -342,5 +346,7 @@ public:
      * @return constant reference to shared pointer object of V plane
      */
     const Blob::Ptr& v() const noexcept;
+
+    Blob::Ptr createROI(const ROI& roi) const override;
 };
 }  // namespace InferenceEngine

inference-engine/include/ie_layouts.h

@@ -318,4 +318,36 @@ private:
     BlockingDesc blockingDesc;
 };
 
+/**
+ * @brief This structure describes ROI data for image-like tensors.
+ */
+struct ROI {
+    size_t id = 0;      //!< ID of a ROI (offset over batch dimension)
+    size_t posX = 0;    //!< W upper left coordinate of ROI
+    size_t posY = 0;    //!< H upper left coordinate of ROI
+    size_t sizeX = 0;   //!< W size of ROI
+    size_t sizeY = 0;   //!< H size of ROI
+
+    ROI() = default;
+
+    ROI(size_t id, size_t posX, size_t posY, size_t sizeX, size_t sizeY) :
+        id(id), posX(posX), posY(posY), sizeX(sizeX), sizeY(sizeY) {
+    }
+};
+
+/**
+ * @brief Creates a TensorDesc object for ROI.
+ *
+ * @param origDesc original TensorDesc object.
+ * @param roi An image ROI object inside of the original object.
+ * @param useOrigMemDesc Flag to use original memory description (strides/offset).
+ *     Should be set if the new TensorDesc describes shared memory.
+ *
+ * @return A newly created TensorDesc object representing ROI.
+ */
+INFERENCE_ENGINE_API_CPP(TensorDesc) make_roi_desc(
+        const TensorDesc& origDesc,
+        const ROI& roi,
+        bool useOrigMemDesc);
+
 }  // namespace InferenceEngine

inference-engine/src/inference_engine/CMakeLists.txt

@@ -20,6 +20,7 @@ set(IE_BASE_SOURCE_FILES
       ${CMAKE_CURRENT_SOURCE_DIR}/cnn_network_ngraph_impl.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/generic_ie.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/blob_factory.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/ie_blob_common.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/ie_data.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/ie_layouts.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/ie_memcpy.cpp

inference-engine/src/inference_engine/ie_blob_common.cpp

@@ -2,61 +2,20 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+#include "ie_blob.h"
+
 #include <memory>
 #include <utility>
 #include <vector>
 
-#include "blob_factory.hpp"
-#include "ie_blob.h"
-#include "ie_compound_blob.h"
-
 namespace InferenceEngine {
 
+Blob::Ptr Blob::createROI(const ROI&) const {
+    THROW_IE_EXCEPTION << "[NOT_IMPLEMENTED] createROI is not implemented for current type of Blob";
+}
+
 Blob::Ptr make_shared_blob(const Blob::Ptr& inputBlob, const ROI& roi) {
-    // reject compound blobs
+    return inputBlob->createROI(roi);
-    if (inputBlob->is<CompoundBlob>()) {
-        THROW_IE_EXCEPTION << "Compound blobs do not support ROI";
-    }
-
-    size_t blkDimsH = roi.sizeY;
-    size_t blkDimsW = roi.sizeX;
-    size_t blkDimsC = inputBlob->getTensorDesc().getDims()[1];
-    size_t blkOffset;
-    SizeVector blkOrder;
-    SizeVector blkDims;
-
-    if (roi.posX + roi.sizeX > inputBlob->getTensorDesc().getDims()[3] ||
-        roi.posY + roi.sizeY > inputBlob->getTensorDesc().getDims()[2]) {
-        THROW_IE_EXCEPTION << "passed ROI coordinates are inconsistent to input size";
-    }
-
-    Layout blobLayout = inputBlob->getTensorDesc().getLayout();
-    switch (blobLayout) {
-    case NCHW: {
-        blkOffset = inputBlob->getTensorDesc().getDims()[3] * roi.posY + roi.posX;
-        blkOrder = {0, 1, 2, 3};
-        blkDims = {1, blkDimsC, blkDimsH, blkDimsW};  // we use BlockingDesc for 1 cropped image only
-    } break;
-    case NHWC: {
-        blkOffset = blkDimsC * (inputBlob->getTensorDesc().getDims()[3] * roi.posY + roi.posX);
-        blkOrder = {0, 2, 3, 1};
-        blkDims = {1, blkDimsH, blkDimsW, blkDimsC};  // we use BlockingDesc for 1 cropped image only
-    } break;
-    default: {
-        THROW_IE_EXCEPTION << "ROI could not be cropped due to inconsistent input layout: " << blobLayout;
-    }
-    }
-
-    // the strides are the same because ROI blob uses the same memory buffer as original input blob.
-    SizeVector blkStrides(inputBlob->getTensorDesc().getBlockingDesc().getStrides());
-
-    SizeVector blkDimsOffsets = {0, 0, 0, 0};  // no offset per dims by default
-
-    BlockingDesc blkDesc(blkDims, blkOrder, blkOffset, blkDimsOffsets, blkStrides);
-    TensorDesc tDesc(inputBlob->getTensorDesc().getPrecision(), {1, blkDimsC, blkDimsH, blkDimsW}, blkDesc);
-    tDesc.setLayout(blobLayout);
-
-    return make_blob_with_precision(tDesc, inputBlob->buffer());
 }
 
 }  // namespace InferenceEngine

inference-engine/src/inference_engine/ie_compound_blob.cpp

@@ -15,7 +15,9 @@
 #include <vector>
 
 namespace InferenceEngine {
+
 namespace {
+
 void verifyNV12BlobInput(const Blob::Ptr& y, const Blob::Ptr& uv) {
     // Y and UV must be valid pointers
     if (y == nullptr || uv == nullptr) {
@@ -189,6 +191,7 @@ void verifyI420BlobInput(const Blob::Ptr& y, const Blob::Ptr& u, const Blob::Ptr
                            << yDims[3] << "(Y plane) and " << vDims[3] << "(V plane)";
     }
 }
+
 }  // anonymous namespace
 
 CompoundBlob::CompoundBlob(): Blob(TensorDesc(Precision::UNSPECIFIED, {}, Layout::ANY)) {}
@@ -272,6 +275,17 @@ Blob::Ptr CompoundBlob::getBlob(size_t i) const noexcept {
     return _blobs[i];
 }
 
+Blob::Ptr CompoundBlob::createROI(const ROI& roi) const {
+    std::vector<Blob::Ptr> roiBlobs;
+    roiBlobs.reserve(_blobs.size());
+
+    for (const auto& blob : _blobs) {
+        roiBlobs.push_back(blob->createROI(roi));
+    }
+
+    return std::make_shared<CompoundBlob>(std::move(roiBlobs));
+}
+
 const std::shared_ptr<IAllocator>& CompoundBlob::getAllocator() const noexcept {
     static std::shared_ptr<IAllocator> _allocator = nullptr;
     return _allocator;
@@ -319,6 +333,19 @@ const Blob::Ptr& NV12Blob::uv() const noexcept {
     return _blobs[1];
 }
 
+Blob::Ptr NV12Blob::createROI(const ROI& roi) const {
+    auto yROI = roi;
+    yROI.sizeX += yROI.sizeX % 2;
+    yROI.sizeY += yROI.sizeY % 2;
+
+    const auto uvROI = ROI(yROI.id, yROI.posX / 2, yROI.posY / 2, yROI.sizeX / 2, yROI.sizeY / 2);
+
+    const auto yRoiBlob = y()->createROI(yROI);
+    const auto uvRoiBlob = uv()->createROI(uvROI);
+
+    return std::make_shared<NV12Blob>(yRoiBlob, uvRoiBlob);
+}
+
 I420Blob::I420Blob(const Blob::Ptr& y, const Blob::Ptr& u, const Blob::Ptr& v) {
     // verify data is correct
     verifyI420BlobInput(y, u, v);
@@ -371,4 +398,18 @@ const Blob::Ptr& I420Blob::v() const noexcept {
     return _blobs[2];
 }
 
+Blob::Ptr I420Blob::createROI(const ROI& roi) const {
+    auto yROI = roi;
+    yROI.sizeX += yROI.sizeX % 2;
+    yROI.sizeY += yROI.sizeY % 2;
+
+    const auto uvROI = ROI(yROI.id, yROI.posX / 2, yROI.posY / 2, yROI.sizeX / 2, yROI.sizeY / 2);
+
+    const auto yRoiBlob = y()->createROI(yROI);
+    const auto uRoiBlob = u()->createROI(uvROI);
+    const auto vRoiBlob = v()->createROI(uvROI);
+
+    return std::make_shared<I420Blob>(yRoiBlob, uRoiBlob, vRoiBlob);
+}
+
 }  // namespace InferenceEngine

inference-engine/src/inference_engine/ie_layouts.cpp

@@ -368,3 +368,117 @@ bool BlockingDesc::operator==(const BlockingDesc& rhs) const {
 bool BlockingDesc::operator!=(const BlockingDesc& rhs) const {
     return !(*this == rhs);
 }
+
+namespace {
+
+struct DimSlice {
+    size_t startInd = 0;
+    size_t size = 0;
+
+    DimSlice() = default;
+
+    DimSlice(size_t startInd, size_t size) :
+        startInd(startInd), size(size) {
+    }
+};
+
+using TensorSlice = std::vector<DimSlice>;
+
+void checkROI(
+        const TensorDesc& origDesc,
+        const TensorSlice& roi) {
+    const auto numDims = origDesc.getDims().size();
+
+    if (roi.size() != numDims) {
+        THROW_IE_EXCEPTION
+            << "ROI num dims " << roi.size() <<
+            " differs from original num dims " << numDims;
+    }
+
+    // TensorDesc stores dimensions in standard layout, as well as roi vector
+    for (size_t dimInd = 0; dimInd < numDims; ++dimInd) {
+        const auto fullSize = origDesc.getDims()[dimInd];
+
+        const auto& roiSlice = roi[dimInd];
+        const auto endInd = roiSlice.startInd + roiSlice.size;
+
+        if (endInd > fullSize) {
+            THROW_IE_EXCEPTION
+                << "ROI [" << roiSlice.startInd << ", " << endInd << ")"
+                << " is out of range " << fullSize
+                << " for dimension " << dimInd;
+        }
+    }
+}
+
+TensorDesc make_roi_desc(
+        const TensorDesc& origDesc,
+        const TensorSlice& roi,
+        bool useOrigMemDesc) {
+    const auto numDims = origDesc.getDims().size();
+
+    checkROI(origDesc, roi);
+
+    const auto origPrecision = origDesc.getPrecision();
+
+    const auto& origBlkDesc = origDesc.getBlockingDesc();
+    const auto& origBlkStrides = origBlkDesc.getStrides();
+    const auto& origBlkOrder = origBlkDesc.getOrder();
+
+    SizeVector roiDims(numDims);
+    SizeVector roiBlkDims(numDims);
+    SizeVector roiBlkDimOffsets = origBlkDesc.getOffsetPaddingToData();
+    size_t roiBlkOffset = origBlkDesc.getOffsetPadding();
+
+    IE_ASSERT(origBlkStrides.size() == numDims);
+    IE_ASSERT(origBlkOrder.size() == numDims);
+    IE_ASSERT(roiBlkDimOffsets.size() == numDims);
+
+    // BlockingDesc stores dimensions in memory order, so we need to use origOrder array.
+    // Offsets in `roi` relates to `origDesc` dimensions, while offsets in `BlockingDesc` relates to top parent tensor dimensions.
+    for (size_t memInd = 0; memInd < numDims; ++memInd) {
+        const auto dimInd = origBlkOrder[memInd];
+        const auto& roiSlice = roi[dimInd];
+
+        roiDims[dimInd] = roiSlice.size;
+        roiBlkDims[memInd] = roiSlice.size;
+        roiBlkDimOffsets[memInd] += roiSlice.startInd;
+        roiBlkOffset += roiSlice.startInd * origBlkStrides[memInd];
+    }
+
+    const auto roiBlkDesc =
+        useOrigMemDesc ?
+            BlockingDesc(roiBlkDims, origBlkOrder, roiBlkOffset, roiBlkDimOffsets, origBlkStrides) :
+            BlockingDesc(roiBlkDims, origBlkOrder);
+
+    const auto roiDesc = TensorDesc(origPrecision, roiDims, roiBlkDesc);
+
+    return roiDesc;
+}
+
+TensorSlice make_roi_slice(
+        const TensorDesc& origDesc,
+        const ROI& roi) {
+    const auto layout = origDesc.getLayout();
+    if (layout != Layout::NCHW && layout != Layout::NHWC) {
+        THROW_IE_EXCEPTION
+            << "Unsupported layout " << layout;
+    }
+
+    TensorSlice roiSlice(4);
+    roiSlice[0] = DimSlice {roi.id, 1};                 // N
+    roiSlice[1] = DimSlice {0, origDesc.getDims()[1]};  // C
+    roiSlice[2] = DimSlice {roi.posY, roi.sizeY};       // H
+    roiSlice[3] = DimSlice {roi.posX, roi.sizeX};       // W
+
+    return roiSlice;
+}
+
+}  // namespace
+
+TensorDesc InferenceEngine::make_roi_desc(
+        const TensorDesc& origDesc,
+        const ROI& roi,
+        bool useOrigMemDesc) {
+    return make_roi_desc(origDesc, make_roi_slice(origDesc, roi), useOrigMemDesc);
+}

inference-engine/tests/unit/inference_engine/ie_blob_test.cpp

@@ -395,3 +395,53 @@ TEST_F(BlobTests, makeRoiBlobWrongSize) {
     InferenceEngine::ROI roi = {0, 1, 1, 4, 4};  // cropped picture with: id = 0, (x,y) = (1,1), sizeX (W) = 4, sizeY (H) = 4
     ASSERT_THROW(make_shared_blob(blob, roi), InferenceEngine::details::InferenceEngineException);
 }
+
+TEST_F(BlobTests, readRoiBlob) {
+    // Create original Blob
+
+    const auto origDesc =
+        InferenceEngine::TensorDesc(
+            InferenceEngine::Precision::I32,
+            {1, 3, 4, 8},
+            InferenceEngine::NCHW);
+
+    const auto origBlob =
+        InferenceEngine::make_shared_blob<int32_t>(origDesc);
+    origBlob->allocate();
+
+    // Fill the original Blob
+
+    {
+        auto origMemory = origBlob->wmap();
+        const auto origPtr = origMemory.as<int32_t*>();
+        ASSERT_NE(nullptr, origPtr);
+
+        for (size_t i = 0; i < origBlob->size(); ++i) {
+            origPtr[i] = i;
+        }
+    }
+
+    // Create ROI Blob
+
+    const auto roi = InferenceEngine::ROI(0, 4, 2, 4, 2);
+
+    const auto roiBlob = InferenceEngine::as<InferenceEngine::MemoryBlob>(origBlob->createROI(roi));
+    ASSERT_NE(nullptr, roiBlob);
+
+    // Read ROI Blob
+
+    {
+        const auto roiOffset = roiBlob->getTensorDesc().getBlockingDesc().getOffsetPadding();
+
+        auto roiMemory = roiBlob->rmap();
+        auto roiPtr = roiMemory.as<const int32_t*>();
+        ASSERT_NE(nullptr, roiPtr);
+
+        // Blob::rmap returns pointer to the original blob start, we have to add ROI offset manually.
+        roiPtr += roiOffset;
+
+        for (size_t i = 0; i < roiBlob->size(); ++i) {
+            ASSERT_EQ(roiPtr[i], i + roiOffset);
+        }
+    }
+}

inference-engine/tests_deprecated/functional/shared_tests/io_blob_tests/cropResize_tests.hpp

@@ -398,6 +398,17 @@ TEST_P(RandomROITest, PreprocRandomROITest)
 
         if (_colorFormat == NV12)
         {
+            if (i % 2)
+            {
+                // New way to create NV12 ROI
+
+                auto nv12Blob = make_shared_blob<NV12Blob>(yBlob, uvBlob);
+                cropBlob = nv12Blob->createROI(roi);
+            }
+            else
+            {
+                // Old way to create NV12 ROI
+
                 roi.sizeX += roi.sizeX % 2;
                 roi.sizeY += roi.sizeY % 2;
 
@@ -408,6 +419,7 @@ TEST_P(RandomROITest, PreprocRandomROITest)
 
                 cropBlob = make_shared_blob<NV12Blob>(cropYBlob, cropUvBlob);
             }
+        }
         else
         {
             cropBlob = make_shared_blob(blob, roi);
@@ -1110,13 +1122,27 @@ TEST_P(NV12ColorConvertTest, NV12Test) {
     cv::resize(refImg, refImg, cv::Size(_netDims[3], _netDims[2]), 0, 0, cv_interpolation);
     auto refBlob = img2Blob<Precision::FP32>(refImg, Layout::NCHW);
 
-    // Note: Y and UV blobs for original data must always be "alive" until the end of the execution:
-    //       ROI blobs do not own the data
     auto yBlob = img2Blob<Precision::U8>(yPlane, NHWC);
     auto uvBlob = img2Blob<Precision::U8>(uvPlane, NHWC);
+    Blob::Ptr inputBlob;
+
+    if (i % 2)
+    {
+        // New way to create NV12 ROI
+
+        auto nv12Blob = make_shared_blob<NV12Blob>(yBlob, uvBlob);
+        inputBlob = nv12Blob->createROI(yRoi);
+    }
+    else
+    {
+        // Old way to create NV12 ROI
+
+        // Note: Y and UV blobs for original data must always be "alive" until the end of the execution:
+        //       ROI blobs do not own the data
         auto croppedYBlob = make_shared_blob(yBlob, yRoi);
         auto croppedUvBlob = make_shared_blob(uvBlob, uvRoi);
-    auto inputBlob = make_shared_blob<NV12Blob>(croppedYBlob, croppedUvBlob);
+        inputBlob = make_shared_blob<NV12Blob>(croppedYBlob, croppedUvBlob);
+    }
 
     req.SetBlob(net.getInputsInfo().begin()->first, inputBlob);