Dynamic plugin reshape blob (#6969)

* Added nGraph as a public dependency

* Fixed Windows warning

* Fixed CMake

* Fixed constant op

* Fixed typo

* Added reshape to PartialShape to CNNNetwork

* Added SetShape to InferRequest

* Enable support of DynamicShape in IE Data

* Add support of dynamic shapes to template plugin

* Fixed build

* Enable support dynamic rank

* Fixed test for dynamic rank

* Fixed some tests

* Fix preprocess tests

* Fixed SetBlob

* Fixed code style

* Add more tests

* Fixed accuracy tests

* Fixed documentation

* Added tests for custom operation

* Added new tests

* WIP: move setShape from infer request to Blob

* Returned isApplicable check back

* Removed obsolete tests for InferRequest::SetShape and add new test for Blob::setShape (a trivial one)

* Fixed artifacts

* Break code style

* Revert "Break code style"

This reverts commit 71ee638cd0.

* Added -j8 for fix_all

* Applied code style fixes

* Adde doxygen items

* Fixed style

* Applied codestyle patch

* Reverted unrelevant commit with template extension

* Fixed cmake file for shared func tests (pick from master)

* Revert all changes in template_extension

* Deleted some old stuff that commented and won't be used

* Fixed wrong exception throwing

* Code style fix

* Fixed preprocessing part

* Fixed incorrect blob reshape in GetBlob

* Deleted incorrect assert in GAPI that prevents passing some tests in Debug

* Fixed issues identified during review

* Removed SetShape, replace getLayoutByDims by getLayoutByRank and removed problematic modification from IE preprocessing

* Fixed comments

* Removed obsolete setShape

* [VPU] Fixed allocating dynamic blobs in myriad_infer_request

* Fixed comments

* Fixed CNNNgraphImpl and comments

Co-authored-by: Ilya Churaev <ilya.churaev@intel.com>
Co-authored-by: Polina <polina.brzezinskaya@intel.com>

inference-engine/src/inference_engine/include/ie/cpp/ie_cnn_network.h

@@ -182,8 +182,20 @@ public:
      * @param inputShapes A map of pairs: name of corresponding data and its dimension.
      */
     void reshape(const ICNNNetwork::InputShapes& inputShapes);
+
+    /**
+     * @brief Run shape inference with new input shapes for the network
+     * @param inputShapes A map of pairs: name of corresponding data and its dimension.
+     */
+    void reshape(const std::initializer_list<ICNNNetwork::InputShapes::value_type>& inputShapes);
     IE_SUPPRESS_DEPRECATED_END
 
+    /**
+     * @brief Run shape inference with new input partial shapes for the network
+     * @param inputShapes A map of pairs: name of corresponding data and its dimension.
+     */
+    void reshape(const std::map<std::string, ngraph::PartialShape>& inputShapes);
+
     /**
      * @brief Serialize network to IR and weights files.
      *

inference-engine/src/inference_engine/include/ie/ie_blob.h

@@ -187,6 +187,28 @@ public:
      */
     virtual bool deallocate() noexcept = 0;
 
+    /**
+     * @brief Set new shape for blob, deallocate/allocate if new total size is bigger than previous one.
+     *
+     * @param dims new shape
+     */
+    void setShape(const SizeVector& dims) {
+        if (properProduct(dims) > properProduct(getTensorDesc().getDims())) {
+            // New blob shape requires more memory than old one -- reallocate
+            if (!deallocate())
+                IE_THROW() << "Cannot deallocate blob while an attempt to enlarge blob area in setShape.";
+
+            // Old and new ranks should match as well as layouts
+            getTensorDesc().setDims(dims);
+
+            allocate();
+            // no way to detect if allocation is successful other than map/unmap that we wouldn't like to do here
+        } else {
+            // Don't shrink area when new size fit the existing area
+            getTensorDesc().setDims(dims);
+        }
+    }
+
     /**
      * @deprecated Cast to MemoryBlob and use new wlock/rwlock API instead.
      * Blob class can represent compound blob, which do not refer to the only solid memory.
@@ -239,6 +261,17 @@ protected:
         return std::accumulate(std::begin(dims), std::end(dims), (size_t)1, std::multiplies<size_t>());
     }
 
+    /**
+     * @deprecated Cast to MemoryBlob and use its API instead.
+     * @brief Multiplies the dimension vector values. Size of a scalar is 1 instead of 0 as for product.
+     *
+     * @param dims Reference to a vector with dimension values of type size_t
+     * @return Result of multiplication
+     */
+    static size_t properProduct(const SizeVector& dims) noexcept {
+        return std::accumulate(std::begin(dims), std::end(dims), (size_t)1, std::multiplies<size_t>());
+    }
+
     /**
      * @brief Gets an allocator for allocator-based blobs
      *

inference-engine/src/inference_engine/include/ie/ie_data.h

@@ -11,6 +11,7 @@
 
 #include <map>
 #include <memory>
+#include <ngraph/ngraph.hpp>
 #include <string>
 #include <vector>
 
@@ -38,6 +39,16 @@ public:
      */
     Data(const std::string& name, Precision _precision, Layout layout = NCHW);
 
+    /**
+     * @brief A constructor with partial shape
+     *
+     * @param name Name of the data node
+     * @param _precision Precision of the data
+     * @param shape Partial shape of the data
+     * @param layout Data layout
+     */
+    Data(const std::string& name, Precision _precision, const ngraph::PartialShape& shape, Layout layout = BLOCKED);
+
     /**
      * @brief A constructor with tensor descriptor
      *
@@ -91,6 +102,20 @@ public:
      * @param layout new layout
      */
     void reshape(const SizeVector& dims, Layout layout);
+    /**
+     * @brief changes dims and layout at same time
+     *
+     * @param dims new dimensions
+     * @param layout new layout
+     */
+    void reshape(const std::initializer_list<size_t>& dims, Layout layout);
+    /**
+     * @brief changes dims and layout at same time
+     *
+     * @param dims new dimensions
+     * @param layout new layout
+     */
+    void reshape(const ngraph::PartialShape& dims, Layout layout);
 
     /**
      * @brief Gets the layout value for this Data instance
@@ -142,6 +167,18 @@ public:
      */
     const UserValue& getUserObject() const;
 
+    /**
+     * @brief Checks if current data has dynamic shapes
+     * @return true if data has dynamic shapes
+     */
+    bool isDynamic() const;
+
+    /**
+     * @brief Returns partial shapes
+     * @return shapes which can have dynamic dimensions
+     */
+    const ngraph::PartialShape& getPartialShape() const;
+
     /**
      * @private
      * @brief Don't touch this field. An implementation details for Data object.
@@ -163,5 +200,7 @@ private:
      * @brief A tensor descriptor
      */
     mutable TensorDesc tensorDesc;
+
+    ngraph::PartialShape pShape;
 };
 }  // namespace InferenceEngine

inference-engine/src/inference_engine/include/ie/ie_icnn_network.hpp

@@ -183,6 +183,22 @@ public:
         return NOT_IMPLEMENTED;
     };
 
+    /**
+     * @deprecated Use InferenceEngine::CNNNetwork wrapper instead
+     * @brief Run shape inference with new input shapes for the network
+     *
+     * @param partialShapes - map of pairs: name of corresponding data and its dimension.
+     * @param resp Pointer to the response message that holds a description of an error if any occurred
+     * @return Status code of the operation
+     */
+    INFERENCE_ENGINE_DEPRECATED("Use InferenceEngine::CNNNetwork wrapper instead")
+    virtual StatusCode reshape(const std::map<std::string, ngraph::PartialShape>& partialShapes,
+                               ResponseDesc* resp) noexcept {
+        (void)partialShapes;
+        (void)resp;
+        return NOT_IMPLEMENTED;
+    };
+
     /**
      * @deprecated Use InferenceEngine::CNNNetwork wrapper instead
      * @brief Serialize network to IR and weights files.

inference-engine/src/inference_engine/include/ie/ie_input_info.hpp

@@ -138,6 +138,17 @@ public:
         return _inputData->getTensorDesc();
     }
 
+    /**
+     * @brief Returns the input shape. May have undefined dimensions.
+     * @return PartialShape object describing input shape.
+     */
+    ngraph::PartialShape getPartialShape() {
+        if (!_inputData) {
+            IE_THROW() << "Data is empty!";
+        }
+        return _inputData->getPartialShape();
+    }
+
     /**
      * @brief Gets pre-process info for the input
      * @return A reference to the PreProcessInfo instance that contains pre-process info for this input

inference-engine/src/inference_engine/src/cnn_network_ngraph_impl.cpp

@@ -65,24 +65,21 @@ void CNNNetworkNGraphImpl::createDataForResult(const ::ngraph::Output<::ngraph::
             return false;
         }
     };
-    // query shape from ngraph::Parameter output shape and check there are no zeros in it
-    SizeVector dims;
-    if (output.get_partial_shape().is_static()) {
-        dims = output.get_shape();
-    }
-    for (const auto& dim : dims) {
-        if (!dim)
+    auto shape = output.get_partial_shape();
+    auto rank = shape.rank().is_static() ? shape.rank().get_length() : 0;
+    for (const auto& dim : shape) {
+        if (dim.is_static() && dim.get_length() == 0)
             IE_THROW() << outName << " has zero dimension which is not allowed";
     }
 
     if (ptr) {
         const auto origLayout = ptr->getTensorDesc().getLayout();
-        const auto layout = isCompatible(dims.size(), origLayout) ? origLayout : TensorDesc::getLayoutByDims(dims);
-        ptr->reshape(dims, layout);
+        const auto layout = isCompatible(rank, origLayout) ? origLayout : TensorDesc::getLayoutByRank(rank);
+        ptr->reshape(shape, layout);
     } else {
-        const auto layout = TensorDesc::getLayoutByDims(dims);
+        const auto layout = TensorDesc::getLayoutByRank(rank);
         const auto precision = details::convertPrecision(output.get_element_type());
-        ptr.reset(new Data(outName, {precision, dims, layout}));
+        ptr.reset(new Data(outName, precision, shape, layout));
     }
 }
 
@@ -180,12 +177,15 @@ CNNNetworkNGraphImpl::CNNNetworkNGraphImpl(const CNNNetwork& network) {
     for (const auto& inputInfo : inputs) {
         InputInfo::Ptr info = std::make_shared<InputInfo>();
         const auto& name = inputInfo.second->getInputData()->getName();
-        DataPtr input = std::make_shared<Data>(name, inputInfo.second->getInputData()->getTensorDesc());
+        const auto& inData = inputInfo.second->getInputData();
+        DataPtr input =
+            std::make_shared<Data>(name, inData->getPrecision(), inData->getPartialShape(), inData->getLayout());
         _data[name] = input;
         info->setInputData(input);
         info->getPreProcess() = inputInfo.second->getPreProcess();
         info->setPrecision(inputInfo.second->getPrecision());
-        info->setLayout(inputInfo.second->getLayout());
+        if (!inData->isDynamic())
+            info->setLayout(inputInfo.second->getLayout());
         _inputData[name] = info;
     }
 }
@@ -296,9 +296,9 @@ size_t CNNNetworkNGraphImpl::getBatchSize() const noexcept {
     });
 
     for (const auto& param : params) {
-        if (param->get_partial_shape().rank().is_dynamic())
+        if (param->get_output_partial_shape(0).rank().is_dynamic())
             continue;
-        auto pshape = param->get_partial_shape();
+        auto pshape = param->get_output_partial_shape(0);
         auto rank = pshape.rank().get_length();
         // WA: for speech recognition and scalar layouts (copy-past from CNNNetwork)
         if ((rank == 2 || rank > 3) && pshape[0].is_static()) {
@@ -312,7 +312,7 @@ void CNNNetworkNGraphImpl::reshape() {
     reshape({});
 }
 
-StatusCode CNNNetworkNGraphImpl::reshape(const std::map<std::string, std::vector<size_t>>& inputShapes,
+StatusCode CNNNetworkNGraphImpl::reshape(const std::map<std::string, ngraph::PartialShape>& inputShapes,
                                          ResponseDesc* responseDesc) noexcept {
     if (inputShapes.empty())
         return OK;
@@ -326,7 +326,7 @@ StatusCode CNNNetworkNGraphImpl::reshape(const std::map<std::string, std::vector
         if (it == inputShapes.end()) {
             continue;
         }
-        if (param->get_partial_shape().is_dynamic() || param->get_shape() != it->second) {
+        if (param->get_output_partial_shape(0).is_dynamic() || param->get_output_partial_shape(0) != it->second) {
             needReshape = true;
             break;
         }
@@ -338,7 +338,7 @@ StatusCode CNNNetworkNGraphImpl::reshape(const std::map<std::string, std::vector
     // save original parameters shape
     std::map<std::string, ngraph::PartialShape> originalInputShapes;
     for (const auto& param : params) {
-        originalInputShapes[param->get_friendly_name()] = param->get_partial_shape();
+        originalInputShapes[param->get_friendly_name()] = param->get_output_partial_shape(0);
     }
 
     try {
@@ -359,6 +359,14 @@ StatusCode CNNNetworkNGraphImpl::reshape(const std::map<std::string, std::vector
     return OK;
 }
 
+StatusCode CNNNetworkNGraphImpl::reshape(const std::map<std::string, std::vector<size_t>>& inputShapes,
+                                         ResponseDesc* responseDesc) noexcept {
+    std::map<std::string, ngraph::PartialShape> shapes;
+    for (const auto& shape : inputShapes)
+        shapes[shape.first] = ngraph::PartialShape(shape.second);
+    return reshape(shapes, responseDesc);
+}
+
 void CNNNetworkNGraphImpl::reshape(const std::map<std::string, ngraph::PartialShape>& inputShapes) {
     OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "CNNNetworkNGraphImpl::reshape");
 
@@ -565,7 +573,7 @@ StatusCode CNNNetworkNGraphImpl::setBatchSize(size_t size, ResponseDesc* respons
             if (i)
                 ss << ", ";
             ss << "\"" << original_parameters[i]->get_friendly_name()
-               << "\": " << original_parameters[i]->get_partial_shape();
+               << "\": " << original_parameters[i]->get_output_partial_shape(0);
         }
 
         // ill-formed logic from the past setBatchSize (we keep it for backward-compatibility)
@@ -575,7 +583,7 @@ StatusCode CNNNetworkNGraphImpl::setBatchSize(size_t size, ResponseDesc* respons
                               [](std::shared_ptr<ngraph::Node> lhs, std::shared_ptr<ngraph::Node> rhs) {
                                   return lhs->get_friendly_name() < rhs->get_friendly_name();
                               });
-        const auto first_parameter_pshape = first_parameter->get_partial_shape();
+        const auto first_parameter_pshape = first_parameter->get_output_partial_shape(0);
         if (first_parameter_pshape.is_dynamic())
             return DescriptionBuffer(PARAMETER_MISMATCH, responseDesc)
                    << "Cannot set batch! Function contains parameter with partially defined shape!" << ss.str();
@@ -587,7 +595,7 @@ StatusCode CNNNetworkNGraphImpl::setBatchSize(size_t size, ResponseDesc* respons
 
         std::map<std::string, std::vector<size_t>> inShapes;
         for (const auto& parameter : original_parameters) {
-            const auto& pshape = parameter->get_partial_shape();
+            const auto& pshape = parameter->get_output_partial_shape(0);
             if (pshape.is_dynamic())
                 return DescriptionBuffer(PARAMETER_MISMATCH, responseDesc)
                        << "Cannot set batch! Function contains parameter with partially defined shape!" << ss.str();

inference-engine/src/inference_engine/src/cnn_network_ngraph_impl.hpp

@@ -74,6 +74,8 @@ public:
 
     StatusCode reshape(const std::map<std::string, std::vector<size_t>>& inputShapes,
                        ResponseDesc* resp) noexcept override;
+    StatusCode reshape(const std::map<std::string, ngraph::PartialShape>& inputShapes,
+                       ResponseDesc* resp) noexcept override;
 
     StatusCode serialize(const std::string& xmlPath, const std::string& binPath, ResponseDesc* resp) const
         noexcept override;

inference-engine/src/inference_engine/src/cpp/ie_cnn_network.cpp

@@ -127,6 +127,14 @@ void CNNNetwork::reshape(const ICNNNetwork::InputShapes& inputShapes) {
     CALL_STATUS_FNC(reshape, inputShapes);
 }
 
+void CNNNetwork::reshape(const std::initializer_list<ICNNNetwork::InputShapes::value_type>& inputShapes) {
+    return reshape(ICNNNetwork::InputShapes(inputShapes));
+}
+
+void CNNNetwork::reshape(const std::map<std::string, ngraph::PartialShape>& inputShapes) {
+    CALL_STATUS_FNC(reshape, inputShapes);
+}
+
 void CNNNetwork::serialize(const std::string& xmlPath, const std::string& binPath) const {
     CALL_STATUS_FNC(serialize, xmlPath, binPath);
 }

inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp

@@ -126,11 +126,8 @@ Blob::Ptr IInferRequestInternal::GetBlob(const std::string& name) {
             data = it->second->getRoiBlob();
         } else {
             data = _inputs[name];
-            checkBlob(
-                data,
-                name,
-                true,
-                foundInput->getTensorDesc().getLayout() != SCALAR ? foundInput->getTensorDesc().getDims() : oneVector);
+            const auto& dims = foundInput->getTensorDesc().getDims();
+            checkBlob(data, name, true, foundInput->getTensorDesc().getLayout() != SCALAR ? dims : oneVector);
 
             auto& devBlob = _deviceInputs[name];
             if (preProcessingRequired(foundInput, data, devBlob)) {
@@ -140,11 +137,8 @@ Blob::Ptr IInferRequestInternal::GetBlob(const std::string& name) {
         }
     } else {
         data = _outputs[name];
-        checkBlob(
-            data,
-            name,
-            false,
-            foundOutput->getTensorDesc().getLayout() != SCALAR ? foundOutput->getTensorDesc().getDims() : oneVector);
+        const auto& dims = foundOutput->getTensorDesc().getDims();
+        checkBlob(data, name, false, foundOutput->getTensorDesc().getLayout() != SCALAR ? dims : oneVector);
     }
     return data;
 }
@@ -252,6 +246,7 @@ void IInferRequestInternal::checkBlob(const Blob::Ptr& blob,
         IE_THROW(NotAllocated) << strNotAllocated;
     }
     size_t refSize;
+    bool isDynamic = false;
     if (refDims.empty()) {
         SizeVector dims;
         if (isInput) {
@@ -263,6 +258,7 @@ void IInferRequestInternal::checkBlob(const Blob::Ptr& blob,
             if (foundInputPair == std::end(_networkInputs)) {
                 IE_THROW(NotFound) << "Failed to find input with name: \'" << name << "\'";
             }
+            isDynamic = foundInputPair->second->getInputData()->getPartialShape().is_dynamic();
             dims = foundInputPair->second->getTensorDesc().getDims();
             refSize = foundInputPair->second->getTensorDesc().getLayout() != SCALAR ? details::product(dims) : 1;
         } else {
@@ -274,14 +270,22 @@ void IInferRequestInternal::checkBlob(const Blob::Ptr& blob,
             if (foundOutputPair == std::end(_networkOutputs)) {
                 IE_THROW(NotFound) << "Failed to find output with name: \'" << name << "\'";
             }
-            dims = foundOutputPair->second->getTensorDesc().getDims();
+            isDynamic = foundOutputPair->second->getPartialShape().is_dynamic();
+            ngraph::PartialShape blobPartialShape(blob->getTensorDesc().getDims());
+            if (foundOutputPair->second->getPartialShape().compatible(blobPartialShape)) {
+                dims = blob->getTensorDesc().getDims();
+            } else {
+                // TODO: it is strange to request tensor desc from data when the shapes are not compatible, probably we
+                // need to immediately throw here
+                dims = foundOutputPair->second->getTensorDesc().getDims();
+            }
             refSize = foundOutputPair->second->getTensorDesc().getLayout() != SCALAR ? details::product(dims) : 1;
         }
     } else {
         refSize = details::product(refDims);
     }
 
-    if (refSize != blob->size()) {
+    if (!isDynamic && refSize != blob->size()) {
         IE_THROW() << strNotMatched + ": got " << blob->size() << " expecting " << refSize;
     }
     const bool remoteBlobPassed = blob->is<RemoteBlob>();

inference-engine/src/inference_engine/src/ie_data.cpp

@@ -77,7 +77,22 @@ Data::Data(const std::string& name, Precision _precision, Layout layout)
     _impl = std::make_shared<Impl>();
 }
 
-Data::Data(const std::string& name, const TensorDesc& desc) : name(name), userObject({0}), tensorDesc(desc) {
+Data::Data(const std::string& name, const TensorDesc& desc)
+    : name(name),
+      userObject({0}),
+      tensorDesc(desc),
+      pShape(desc.getDims()) {
+    _impl = std::make_shared<Impl>();
+}
+
+Data::Data(const std::string& name, Precision _precision, const ngraph::PartialShape& shape, Layout layout)
+    : name(name),
+      userObject({0}),
+      tensorDesc(_precision, layout),
+      pShape(shape) {
+    if (pShape.is_static()) {
+        tensorDesc.reshape(pShape.to_shape(), tensorDesc.getLayout());
+    }
     _impl = std::make_shared<Impl>();
 }
 
@@ -95,6 +110,15 @@ bool Data::isInitialized() const {
 
 void Data::setDims(const SizeVector& a_dims) {
     tensorDesc.setDims(a_dims);
+    pShape = ngraph::PartialShape(a_dims);
+}
+
+bool Data::isDynamic() const {
+    return tensorDesc.getDims().empty() && tensorDesc.getLayout() != SCALAR && pShape.is_dynamic();
+}
+
+const ngraph::PartialShape& Data::getPartialShape() const {
+    return pShape;
 }
 
 void Data::setLayout(Layout layout) {
@@ -103,9 +127,27 @@ void Data::setLayout(Layout layout) {
 
 void Data::reshape(const SizeVector& a_dims, Layout a_layout) {
     tensorDesc.reshape(a_dims, a_layout);
+    pShape = ngraph::PartialShape(a_dims);
 }
 
-Data::Data(const Data& data) : name(data.name), userObject(data.userObject), tensorDesc(data.tensorDesc) {
+void Data::reshape(const std::initializer_list<size_t>& dims, Layout layout) {
+    reshape(SizeVector(dims), layout);
+}
+
+void Data::reshape(const ngraph::PartialShape& dims, Layout layout) {
+    if (dims.is_static()) {
+        reshape(SizeVector(dims.to_shape()), layout);
+    } else {
+        tensorDesc = TensorDesc(tensorDesc.getPrecision(), layout);
+        pShape = dims;
+    }
+}
+
+Data::Data(const Data& data)
+    : name(data.name),
+      userObject(data.userObject),
+      tensorDesc(data.tensorDesc),
+      pShape(data.pShape) {
     _impl = std::make_shared<Impl>();
     _impl->creatorLayer = data._impl->creatorLayer;
     _impl->inputTo = data._impl->inputTo;
@@ -116,6 +158,7 @@ Data& Data::operator=(const Data& data) {
         name = data.name;
         userObject = data.userObject;
         tensorDesc = data.tensorDesc;
+        pShape = data.pShape;
 
         _impl->creatorLayer = data._impl->creatorLayer;
         _impl->inputTo = data._impl->inputTo;
@@ -145,6 +188,11 @@ void Data::setPrecision(const Precision& precision) {
 }
 
 const SizeVector& Data::getDims() const {
+    if (isDynamic())
+        IE_THROW() << "Cannot return dims for Data with dynamic shapes!";
+    if (tensorDesc.getDims().empty() && tensorDesc.getLayout() != SCALAR) {
+        tensorDesc.setDims(pShape.to_shape());
+    }
     return tensorDesc.getDims();
 }
 

inference-engine/src/preprocessing/ie_preprocess_gapi_kernels_simd_impl.hpp

@@ -156,7 +156,6 @@ CV_ALWAYS_INLINE void splitRowC3_Impl(const T in[], T out0[],
 
 #if MANUAL_SIMD
     constexpr int nlanes = VecT::nlanes;
-    GAPI_DbgAssert(length >= nlanes);
 
     VecT r0, r1, r2;
     for (; length >= nlanes;) {

inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp

@@ -266,7 +266,7 @@ void MyriadInferRequest::GetResult() {
                 "Can not find tensor descriptor by plugin for {} output", ieBlobName);
             const auto& dynOutputDesc = descFromPlugin->second;
 
-            if (ieBlob->getTensorDesc().getLayout() != dynOutputDesc.getLayout()) {
+            if (ieBlob->getTensorDesc().getDims() != dynOutputDesc.getDims()) {
                 ieBlob->deallocate();
                 ieBlob->getTensorDesc().reshape(dynOutputDesc.getDims(), dynOutputDesc.getLayout());
                 ieBlob->allocate();

inference-engine/tests/functional/inference_engine/ngraph_reshape_tests.cpp

@@ -70,13 +70,15 @@ TEST_F(NGraphReshapeTests, ReshapedDynamicShapeLayout) {
     }
 
     CNNNetwork cnnNetwork(ngraph);
-    ASSERT_EQ(Layout::SCALAR, cnnNetwork.getInputsInfo()["A"]->getLayout());
+    ASSERT_EQ(Layout::NCHW, cnnNetwork.getInputsInfo()["A"]->getLayout());
+    ASSERT_TRUE(cnnNetwork.getInputsInfo()["A"]->getInputData()->isDynamic());
 
     ICNNNetwork::InputShapes new_shape;
     new_shape["A"] = ngraph::Shape{1, 3, 22, 22};
     cnnNetwork.reshape(new_shape);
 
     ASSERT_EQ(Layout::NCHW, cnnNetwork.getInputsInfo()["A"]->getLayout());
+    ASSERT_FALSE(cnnNetwork.getInputsInfo()["A"]->getInputData()->isDynamic());
 }
 
 TEST_F(NGraphReshapeTests, ReshapeBatchReLU) {
@@ -236,6 +238,120 @@ TEST_F(NGraphReshapeTests, CNNReshapeSpatialReLUWithoutCloneFunction) {
     ASSERT_EQ(ngraph->get_results()[0]->get_shape(), ngraph::Shape({1, 3, 25, 25}));
 }
 
+TEST_F(NGraphReshapeTests, CNNReshapeSpatialReLUStaticToDynamic) {
+    const ngraph::PartialShape refShape{1, 3, ngraph::Dimension::dynamic(), 25};
+    std::shared_ptr<ngraph::Function> ngraph;
+    {
+        ngraph::PartialShape shape({1, 3, 22, 22});
+        ngraph::element::Type type(ngraph::element::Type_t::f32);
+        auto param = std::make_shared<ngraph::op::Parameter>(type, shape);
+        param->set_friendly_name("data");
+        auto relu = std::make_shared<ngraph::op::Relu>(param);
+        auto result = std::make_shared<ngraph::op::Result>(relu);
+
+        ngraph::ParameterVector params = {param};
+        ngraph::ResultVector results = {result};
+
+        ngraph = std::make_shared<ngraph::Function>(results, params);
+    }
+
+    ASSERT_EQ(ngraph->get_parameters()[0]->get_shape(), ngraph::Shape({1, 3, 22, 22}));
+    ASSERT_EQ(ngraph->get_results()[0]->get_shape(), ngraph::Shape({1, 3, 22, 22}));
+
+    CNNNetwork cnnNetwork(ngraph);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes["data"] = refShape;
+
+    ASSERT_NO_THROW(cnnNetwork.reshape(shapes));
+
+    auto changedFunction = cnnNetwork.getFunction();
+    ASSERT_NE(nullptr, changedFunction);
+    ASSERT_TRUE(changedFunction->get_parameters()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(changedFunction->get_results()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(ngraph->get_parameters()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(ngraph->get_results()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_EQ(changedFunction->get_parameters()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(changedFunction->get_results()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(ngraph->get_parameters()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(ngraph->get_results()[0]->get_output_partial_shape(0), refShape);
+}
+
+TEST_F(NGraphReshapeTests, CNNReshapeSpatialReLUStaticToFullyDynamic) {
+    const ngraph::PartialShape refShape = ngraph::PartialShape::dynamic();
+    std::shared_ptr<ngraph::Function> ngraph;
+    {
+        ngraph::PartialShape shape({1, 3, 22, 22});
+        ngraph::element::Type type(ngraph::element::Type_t::f32);
+        auto param = std::make_shared<ngraph::op::Parameter>(type, shape);
+        param->set_friendly_name("data");
+        auto relu = std::make_shared<ngraph::op::Relu>(param);
+        auto result = std::make_shared<ngraph::op::Result>(relu);
+
+        ngraph::ParameterVector params = {param};
+        ngraph::ResultVector results = {result};
+
+        ngraph = std::make_shared<ngraph::Function>(results, params);
+    }
+
+    ASSERT_EQ(ngraph->get_parameters()[0]->get_shape(), ngraph::Shape({1, 3, 22, 22}));
+    ASSERT_EQ(ngraph->get_results()[0]->get_shape(), ngraph::Shape({1, 3, 22, 22}));
+
+    CNNNetwork cnnNetwork(ngraph);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes["data"] = refShape;
+
+    ASSERT_NO_THROW(cnnNetwork.reshape(shapes));
+
+    auto changedFunction = cnnNetwork.getFunction();
+    ASSERT_NE(nullptr, changedFunction);
+    ASSERT_TRUE(changedFunction->get_parameters()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(changedFunction->get_results()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(ngraph->get_parameters()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(ngraph->get_results()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_EQ(changedFunction->get_parameters()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(changedFunction->get_results()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(ngraph->get_parameters()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(ngraph->get_results()[0]->get_output_partial_shape(0), refShape);
+}
+
+TEST_F(NGraphReshapeTests, CNNReshapeSpatialReLUDynamicToDynamic) {
+    const ngraph::PartialShape refShape{1, 3, ngraph::Dimension::dynamic(), 25};
+    std::shared_ptr<ngraph::Function> ngraph;
+    {
+        ngraph::PartialShape shape({1, 3, 22, ngraph::Dimension::dynamic()});
+        ngraph::element::Type type(ngraph::element::Type_t::f32);
+        auto param = std::make_shared<ngraph::op::Parameter>(type, shape);
+        param->set_friendly_name("data");
+        auto relu = std::make_shared<ngraph::op::Relu>(param);
+        auto result = std::make_shared<ngraph::op::Result>(relu);
+
+        ngraph::ParameterVector params = {param};
+        ngraph::ResultVector results = {result};
+
+        ngraph = std::make_shared<ngraph::Function>(results, params);
+    }
+
+    ASSERT_EQ(ngraph->get_parameters()[0]->get_output_partial_shape(0), ngraph::PartialShape({1, 3, 22, ngraph::Dimension::dynamic()}));
+    ASSERT_EQ(ngraph->get_results()[0]->get_output_partial_shape(0), ngraph::PartialShape({1, 3, 22, ngraph::Dimension::dynamic()}));
+
+    CNNNetwork cnnNetwork(ngraph);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes["data"] = refShape;
+
+    ASSERT_NO_THROW(cnnNetwork.reshape(shapes));
+
+    auto changedFunction = cnnNetwork.getFunction();
+    ASSERT_NE(nullptr, changedFunction);
+    ASSERT_TRUE(changedFunction->get_parameters()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(changedFunction->get_results()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(ngraph->get_parameters()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_TRUE(ngraph->get_results()[0]->get_output_partial_shape(0).is_dynamic());
+    ASSERT_EQ(changedFunction->get_parameters()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(changedFunction->get_results()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(ngraph->get_parameters()[0]->get_output_partial_shape(0), refShape);
+    ASSERT_EQ(ngraph->get_results()[0]->get_output_partial_shape(0), refShape);
+}
+
 class CustomTestOp: public ngraph::op::Op {
 public:
     static constexpr ngraph::NodeTypeInfo type_info{"CustomTestLayer", 0};

inference-engine/tests/functional/plugin/shared/include/behavior/infer_request.hpp

@@ -0,0 +1,656 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <chrono>
+#include <future>
+#include <tuple>
+#include <vector>
+#include <string>
+#include <memory>
+#include "ie_extension.h"
+#include <condition_variable>
+#include "shared_test_classes/base/layer_test_utils.hpp"
+#include "ngraph_functions/utils/ngraph_helpers.hpp"
+#include "ngraph_functions/builders.hpp"
+#include <string>
+#include <ie_core.hpp>
+#include <thread>
+#include <base/behavior_test_utils.hpp>
+#include "common_test_utils/common_utils.hpp"
+#include "functional_test_utils/plugin_cache.hpp"
+#include "functional_test_utils/blob_utils.hpp"
+#include "ngraph_functions/subgraph_builders.hpp"
+#include "shared_test_classes/subgraph/basic_lstm.hpp"
+
+namespace BehaviorTestsDefinitions {
+using InferRequestTests = BehaviorTestsUtils::BehaviorTestsBasic;
+
+// Setting empty config to LoadNetwork doesn't throw
+TEST_P(InferRequestTests, SetEmptyConfig) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    InferenceEngine::ExecutableNetwork execNet;
+    std::map<std::string, std::string> config {};
+    if (targetDevice.find(CommonTestUtils::DEVICE_AUTO) == std::string::npos &&
+        targetDevice.find(CommonTestUtils::DEVICE_MULTI) == std::string::npos &&
+        targetDevice.find(CommonTestUtils::DEVICE_HETERO) == std::string::npos) {
+        ASSERT_NO_THROW(ie->SetConfig(configuration, targetDevice));
+        ASSERT_NO_THROW(execNet = ie->LoadNetwork(cnnNet, targetDevice, config));
+    } else {
+        ASSERT_NO_THROW(ie->SetConfig(configuration, targetDevice));
+        ASSERT_NO_THROW(execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration));
+    }
+}
+
+// Load correct network to Plugin to get executable network
+TEST_P(InferRequestTests, canLoadCorrectNetworkToGetExecutable) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    InferenceEngine::ExecutableNetwork execNet;
+    ASSERT_NO_THROW(execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration));
+}
+
+TEST_P(InferRequestTests,  CanCreateTwoExeNetworks) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    InferenceEngine::ExecutableNetwork execNet;
+    for (auto i = 0; i < 2; i++) {
+        ASSERT_NO_THROW(execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration));
+        ASSERT_NE(nullptr, cnnNet.getFunction());
+    }
+}
+
+TEST_P(InferRequestTests, CanCreateInferRequest) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+}
+
+TEST_P(InferRequestTests, failToSetNullptrForInput) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::Blob::Ptr inputBlob = nullptr;
+    ASSERT_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, inputBlob),
+            InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, failToSetEmptyInputBlob) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, blob),
+            InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, failToSetEmptyOutputBlob) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_THROW(req.SetBlob(cnnNet.getOutputsInfo().begin()->first, blob),
+            InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, failToSetNotAllocatedInput) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getInputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, blob));
+}
+
+TEST_P(InferRequestTests, failToSetNotAllocatedOutput) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getOutputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getOutputsInfo().begin()->first, blob));
+}
+
+TEST_P(InferRequestTests, failToSetBlobWithIncorrectName) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    const char incorrect_input_name[] = "incorrect_input_name";
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getInputsInfo().begin()->second->getTensorDesc());
+    blob->allocate();
+    ASSERT_THROW(req.SetBlob(incorrect_input_name, blob),
+            InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, failToSetInputWithIncorrectSizes) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getInputsInfo().begin()->second->getTensorDesc());
+    blob->allocate();
+    blob->getTensorDesc().getDims()[0]*=2;
+    ASSERT_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, blob),
+            InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, failToSetOutputWithIncorrectSizes) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getOutputsInfo().begin()->second->getTensorDesc());
+    blob->allocate();
+    blob->getTensorDesc().getDims()[0]*=2;
+    ASSERT_THROW(req.SetBlob(cnnNet.getOutputsInfo().begin()->first, blob),
+            InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, canInferWithoutSetAndGetInOut) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(req.Infer());
+}
+
+TEST_P(InferRequestTests, canProcessDeallocatedInputBlobAfterGetBlob) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(req.Infer());
+}
+
+TEST_P(InferRequestTests, canProcessDeallocatedInputBlobAfterGetBlobForAsync) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(req.Infer());
+    ASSERT_NO_THROW(req.StartAsync());
+}
+
+TEST_P(InferRequestTests, canProcessDeallocatedInputBlobAfterGetAndSetBlob) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, blob));
+    ASSERT_NO_THROW(req.Infer());
+    ASSERT_NO_THROW(req.StartAsync());
+}
+
+TEST_P(InferRequestTests, canProcessDeallocatedInputBlobAfterSetBlob) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getInputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    blob->allocate();
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, blob));
+    blob->deallocate();
+    ASSERT_THROW(req.Infer(), InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, canProcessDeallocatedOutputBlobAfterGetBlob) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getOutputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    blob->allocate();
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getOutputsInfo().begin()->first, blob));
+    blob->deallocate();
+    ASSERT_THROW(req.Infer(), InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, canProcessDeallocatedOutputBlobAfterGetBlobForAsync) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getOutputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    blob->allocate();
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getOutputsInfo().begin()->first, blob));
+    blob->deallocate();
+    ASSERT_THROW(req.Infer(), InferenceEngine::Exception);
+    ASSERT_THROW({ req.StartAsync(); req.Wait(); }, InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, canProcessDeallocatedOutputBlobAfterGetAndSetBlob) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getOutputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    blob->allocate();
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getOutputsInfo().begin()->first, blob));
+    blob->deallocate();
+    ASSERT_THROW(req.Infer(), InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, canProcessDeallocatedOutputBlobAfterSetBlob) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getOutputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    blob->allocate();
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getOutputsInfo().begin()->first, blob));
+    blob->deallocate();
+    ASSERT_THROW(req.Infer(), InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, secondCallGetOutputDoNotReAllocateData) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob1;
+    InferenceEngine::Blob::Ptr blob2;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob1 = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(blob2 = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_EQ(blob1.get(), blob2.get());
+}
+
+TEST_P(InferRequestTests, CorrectOneAsyncInferWithGetInOutWithInfWait) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    req.Infer();
+    req.StartAsync();
+    InferenceEngine::StatusCode sts;
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(InferenceEngine::StatusCode::OK, sts);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+}
+
+// Plugin correct infer request with allocating input and result BlobMaps inside plugin
+TEST_P(InferRequestTests, canStartAsyncInferWithGetInOutWithStatusOnlyWait) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getOutputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    req.Infer();
+    req.StartAsync();
+    InferenceEngine::StatusCode sts;
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::STATUS_ONLY);
+    ASSERT_TRUE(sts == InferenceEngine::StatusCode::OK ||
+        sts == InferenceEngine::StatusCode::RESULT_NOT_READY);
+}
+
+// Plugin correct infer request with allocating input and result BlobMaps inside plugin
+TEST_P(InferRequestTests, FailedAsyncInferWithNegativeTimeForWait) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob =
+            FuncTestUtils::createAndFillBlob(cnnNet.getOutputsInfo().begin()->second->getTensorDesc());
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    req.Infer();
+    req.StartAsync();
+    ASSERT_THROW(req.Wait(-2), InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestTests, canRun3SyncRequestsConsistentlyFromThreads) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    auto req1 = execNet.CreateInferRequest();
+    auto req2 = execNet.CreateInferRequest();
+    auto req3 = execNet.CreateInferRequest();
+
+
+    auto f1 = std::async(std::launch::async, [&] { req1.Infer();});
+    auto f2 = std::async(std::launch::async, [&] { req2.Infer();});
+    auto f3 = std::async(std::launch::async, [&] { req3.Infer();});
+
+    ASSERT_NO_THROW(f1.get());
+    ASSERT_NO_THROW(f2.get());
+    ASSERT_NO_THROW(f3.get());
+}
+
+TEST_P(InferRequestTests, canRun3AsyncRequestsConsistentlyWithWait) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    auto req1 = execNet.CreateInferRequest();
+    auto req2 = execNet.CreateInferRequest();
+    auto req3 = execNet.CreateInferRequest();
+
+    req1.StartAsync();
+    ASSERT_NO_THROW(req1.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY));
+
+    req2.Infer();
+    ASSERT_NO_THROW(req2.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY));
+
+    req3.Infer();
+    ASSERT_NO_THROW(req3.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY));
+}
+
+TEST_P(InferRequestTests, canRun3AsyncRequestsConsistentlyFromThreadsWithoutWait) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    auto req1 = execNet.CreateInferRequest();
+    auto req2 = execNet.CreateInferRequest();
+    auto req3 = execNet.CreateInferRequest();
+    InferenceEngine::ResponseDesc response1, response2, response3;
+    InferenceEngine::StatusCode sts1, sts2, sts3;
+
+    req1.Infer();
+    req2.Infer();
+    req3.Infer();
+
+    std::thread t1([&] { req1.StartAsync(); sts1 = req1.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY); });
+    std::thread t2([&] { req2.StartAsync(); sts2 = req2.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY); });
+    std::thread t3([&] { req3.StartAsync(); sts3 = req3.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY); });
+
+    t1.join();
+    t2.join();
+    t3.join();
+
+    ASSERT_EQ(static_cast<int>(InferenceEngine::StatusCode::OK), sts1) << response1.msg;
+    ASSERT_EQ(static_cast<int>(InferenceEngine::StatusCode::OK), sts2) << response2.msg;
+    ASSERT_EQ(static_cast<int>(InferenceEngine::StatusCode::OK), sts3) << response3.msg;
+}
+
+TEST_P(InferRequestTests, canWaitWithotStartAsync) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    auto req = execNet.CreateInferRequest();
+    ASSERT_NO_THROW(req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY));
+    ASSERT_NO_THROW(req.Wait(InferenceEngine::InferRequest::WaitMode::STATUS_ONLY));
+    ASSERT_NO_THROW(req.Wait(1));
+}
+
+TEST_P(InferRequestTests, returnDeviceBusyOnSetBlobAfterAsyncInfer) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    auto&& config = configuration;
+    auto itConfig = config.find(CONFIG_KEY(CPU_THROUGHPUT_STREAMS));
+    if (itConfig != config.end()) {
+        if (itConfig->second != "CPU_THROUGHPUT_AUTO") {
+            if (std::stoi(itConfig->second) == 0) {
+                GTEST_SKIP() << "Not applicable with disabled streams";
+            }
+        }
+    }
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    auto req = execNet.CreateInferRequest();
+    auto outputBlob = req.GetBlob(cnnNet.getInputsInfo().begin()->first);
+    InferenceEngine::ResponseDesc response;
+
+    InferenceEngine::StatusCode sts;
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::STATUS_ONLY);
+    ASSERT_EQ(InferenceEngine::StatusCode::INFER_NOT_STARTED, sts) << response.msg;
+    req.StartAsync();
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(static_cast<int>(InferenceEngine::StatusCode::OK), sts) << response.msg;
+    try {
+        req.SetBlob(cnnNet.getInputsInfo().begin()->first, outputBlob);
+    }
+    catch (const std::exception &e) {
+        std::cout << "Exception: " << e.what() << std::endl;
+    }
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::STATUS_ONLY);
+    ASSERT_TRUE(sts == InferenceEngine::StatusCode::OK ||
+    sts == InferenceEngine::StatusCode::RESULT_NOT_READY) << response.msg;
+}
+
+TEST_P(InferRequestTests, returnDeviceBusyOnGetBlobAfterAsyncInfer) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    auto req = execNet.CreateInferRequest();
+    auto outputBlob = req.GetBlob(cnnNet.getInputsInfo().begin()->first);
+    InferenceEngine::ResponseDesc response;
+    InferenceEngine::StatusCode sts;
+    req.StartAsync();
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(static_cast<int>(InferenceEngine::StatusCode::OK), sts) << response.msg;
+    try {
+        req.SetBlob(cnnNet.getInputsInfo().begin()->first, outputBlob);
+    }
+    catch (const std::exception &e) {
+        std::cout << "Exception" << e.what() << std::endl;
+    }
+}
+
+TEST_P(InferRequestTests, returnDeviceBusyOnGetPerformanceCountAfterAsyncInfer) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    auto req = execNet.CreateInferRequest();
+    auto outputBlob = req.GetBlob(cnnNet.getInputsInfo().begin()->first);
+    InferenceEngine::ResponseDesc response;
+    InferenceEngine::StatusCode sts;
+    req.StartAsync();
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(static_cast<int>(InferenceEngine::StatusCode::OK), sts) << response.msg;
+
+    std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> perfMap;
+
+    try {
+        perfMap = req.GetPerformanceCounts();
+    }
+    catch (const std::exception &e) {
+        std::cout << "Exception" << e.what() << std::endl;
+    }
+}
+
+class InferRequestTestsResultNotReady : public InferRequestTests {
+};
+
+TEST_P(InferRequestTestsResultNotReady, ReturnResultNotReadyFromWaitInAsyncModeForTooSmallTimeout) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngraph::Function
+    // return ngrpah::Function
+    // GetNetwork(3000, 380) make inference around 20ms on GNA SW
+    // so increases chances for getting RESULT_NOT_READY
+    function = SubgraphTestsDefinitions::Basic_LSTM_S::GetNetwork(300, 38);
+    InferenceEngine::CNNNetwork cnnNet(function);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    InferenceEngine::StatusCode sts = InferenceEngine::StatusCode::OK;
+    std::promise<std::chrono::system_clock::time_point> callbackTimeStamp;
+    auto callbackTimeStampFuture = callbackTimeStamp.get_future();
+    // add a callback to the request and capture the timestamp
+    req.SetCompletionCallback([&]() {
+        callbackTimeStamp.set_value(std::chrono::system_clock::now());
+        });
+    req.StartAsync();
+    ASSERT_NO_THROW(sts = req.Wait(InferenceEngine::InferRequest::WaitMode::STATUS_ONLY));
+    // get timestamp taken AFTER return from the Wait(STATUS_ONLY)
+    const auto afterWaitTimeStamp = std::chrono::system_clock::now();
+    // IF the callback timestamp is larger than the afterWaitTimeStamp
+    // then we should observe RESULT_NOT_READY
+    if (afterWaitTimeStamp < callbackTimeStampFuture.get()) {
+        ASSERT_TRUE(sts == InferenceEngine::StatusCode::RESULT_NOT_READY);
+    }
+    ASSERT_NO_THROW(req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY));
+}
+}  // namespace BehaviorTestsDefinitions

inference-engine/tests/functional/plugin/shared/include/behavior/infer_request_dynamic.hpp

@@ -0,0 +1,320 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <chrono>
+#include <future>
+#include <tuple>
+#include <vector>
+#include <string>
+#include <memory>
+#include "ie_extension.h"
+#include <condition_variable>
+#include "shared_test_classes/base/layer_test_utils.hpp"
+#include "ngraph_functions/utils/ngraph_helpers.hpp"
+#include "ngraph_functions/builders.hpp"
+#include <ngraph/opsets/opset6.hpp>
+#include <string>
+#include <ie_core.hpp>
+#include <thread>
+#include <base/behavior_test_utils.hpp>
+#include "common_test_utils/common_utils.hpp"
+#include "functional_test_utils/plugin_cache.hpp"
+#include "functional_test_utils/blob_utils.hpp"
+#include "ngraph_functions/subgraph_builders.hpp"
+#include "shared_test_classes/subgraph/basic_lstm.hpp"
+
+
+namespace BehaviorTestsDefinitions {
+
+class InferRequestDynamicTests : public BehaviorTestsUtils::BehaviorTestsBasic {
+public:
+    void SetUp()  override {
+        std::tie(netPrecision, targetDevice, configuration) = this->GetParam();
+        function = ngraph::builder::subgraph::makeSplitConvConcat();
+    }
+};
+
+TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithoutSetShape) {
+    const std::string param_name = "Param_1";
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension::dynamic(), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+}
+
+TEST_P(InferRequestDynamicTests, InferDynamicNetworkBoundWithoutSetShape) {
+    const std::string param_name = "Param_1";
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension(0, 5), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+}
+
+
+TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetBlob) {
+    const std::string param_name = "Param_1";
+    const InferenceEngine::SizeVector refShape = {1, 4, 20, 20};
+    const InferenceEngine::SizeVector refOutShape = {1, 10, 18, 18};
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension::dynamic(), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    //ASSERT_NO_THROW(req.SetShape(param_name, {1, 4, 20, 20}));
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(blob->setShape({1, 4, 20, 20}));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape);
+    req.Infer();
+    req.StartAsync();
+    InferenceEngine::StatusCode sts;
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(InferenceEngine::StatusCode::OK, sts);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refOutShape);
+}
+
+TEST_P(InferRequestDynamicTests, InferUpperBoundNetworkWithGetBlob) {
+    const std::string param_name = "Param_1";
+    const InferenceEngine::SizeVector refShape = {1, 4, 20, 20};
+    const InferenceEngine::SizeVector refOutShape = {1, 10, 18, 18};
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension(0, 19), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    //ASSERT_NO_THROW(req.SetShape(param_name, {1, 4, 20, 20}));
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(blob->setShape({1, 4, 20, 20}));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape);
+    req.Infer();
+    req.StartAsync();
+    InferenceEngine::StatusCode sts;
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(InferenceEngine::StatusCode::OK, sts);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refOutShape);
+}
+
+TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetBlobLower) {
+    const std::string param_name = "Param_1";
+    const InferenceEngine::SizeVector refShape = {1, 4, 20, 20};
+    const InferenceEngine::SizeVector refOutShape = {1, 10, 18, 18};
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension(2, 3), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(blob->setShape({1, 4, 20, 20}));
+    // Plugin may or may not throw in case if input tensor has dimensions that are out of bounds
+    //ASSERT_THROW(req.Infer(), InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetBlobUpper) {
+    const std::string param_name = "Param_1";
+    const InferenceEngine::SizeVector refShape = {1, 4, 20, 20};
+    const InferenceEngine::SizeVector refOutShape = {1, 10, 18, 18};
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension(1, 2), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(blob->setShape({3, 4, 20, 20}));
+    // Plugin may or may not throw in case if input tensor has dimensions that are out of bounds
+    // ASSERT_THROW(req.Infer(), InferenceEngine::Exception);
+}
+
+TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetBlob2times) {
+    const std::string param_name = "Param_1";
+    const InferenceEngine::SizeVector refShape = {1, 4, 20, 20};
+    const InferenceEngine::SizeVector refShape2 = {2, 4, 20, 20};
+    const InferenceEngine::SizeVector refOutShape = {1, 10, 18, 18};
+    const InferenceEngine::SizeVector refOutShape2 = {2, 10, 18, 18};
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension::dynamic(), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(blob->setShape(refShape));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape);
+    req.Infer();
+    req.StartAsync();
+    InferenceEngine::StatusCode sts;
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(InferenceEngine::StatusCode::OK, sts);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refOutShape);
+
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(blob->setShape(refShape2));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape2);
+    req.Infer();
+    req.StartAsync();
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(InferenceEngine::StatusCode::OK, sts);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refOutShape2);
+}
+
+
+TEST_P(InferRequestDynamicTests, GetSameBlob2times) {
+    const std::string param_name = "Param_1";
+    const InferenceEngine::SizeVector refShape = {1, 4, 20, 20};
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension::dynamic(), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_NO_THROW(blob->setShape(refShape));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getInputsInfo().begin()->first));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape);
+}
+
+TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetBlob) {
+    const std::string param_name = "Param_1";
+    const InferenceEngine::SizeVector refShape = {1, 4, 20, 20};
+    const InferenceEngine::SizeVector refOutShape = {1, 10, 18, 18};
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension::dynamic(), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob = make_blob_with_precision({InferenceEngine::Precision::FP32, refShape, InferenceEngine::Layout::NCHW});
+    blob->allocate();
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, blob));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape);
+    req.Infer();
+    req.StartAsync();
+    InferenceEngine::StatusCode sts;
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(InferenceEngine::StatusCode::OK, sts);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refOutShape);
+}
+
+TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetBlob2times) {
+    const std::string param_name = "Param_1";
+    const InferenceEngine::SizeVector refShape = {1, 4, 20, 20};
+    const InferenceEngine::SizeVector refShape2 = {2, 4, 20, 20};
+    const InferenceEngine::SizeVector refOutShape = {1, 10, 18, 18};
+    const InferenceEngine::SizeVector refOutShape2 = {2, 10, 18, 18};
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED()
+    // Create CNNNetwork from ngrpah::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, ngraph::PartialShape> shapes;
+    shapes[param_name] = {ngraph::Dimension::dynamic(), 4, 20, 20};
+    cnnNet.reshape(shapes);
+    // Load CNNNetwork to target plugins
+    auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
+    // Create InferRequest
+    InferenceEngine::InferRequest req;
+    InferenceEngine::Blob::Ptr blob = make_blob_with_precision({InferenceEngine::Precision::FP32, refShape, InferenceEngine::Layout::NCHW});
+    blob->allocate();
+
+    ASSERT_NO_THROW(req = execNet.CreateInferRequest());
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, blob));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape);
+    req.Infer();
+    req.StartAsync();
+    InferenceEngine::StatusCode sts;
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(InferenceEngine::StatusCode::OK, sts);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refOutShape);
+
+    blob = make_blob_with_precision({InferenceEngine::Precision::FP32, refShape2, InferenceEngine::Layout::NCHW});
+    blob->allocate();
+    ASSERT_NO_THROW(req.SetBlob(cnnNet.getInputsInfo().begin()->first, blob));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refShape2);
+    req.Infer();
+    req.StartAsync();
+    sts = req.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+    ASSERT_EQ(InferenceEngine::StatusCode::OK, sts);
+    ASSERT_NO_THROW(blob = req.GetBlob(cnnNet.getOutputsInfo().begin()->first));
+    ASSERT_EQ(blob->getTensorDesc().getDims(), refOutShape2);
+}
+
+}  // namespace BehaviorTestsDefinitions

inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/subgraph_builders.hpp

@@ -42,6 +42,7 @@ inline std::shared_ptr<ngraph::Function> makeConvPoolRelu(std::vector<size_t> in
 inline std::shared_ptr<ngraph::Function> makeSplitConvConcat(std::vector<size_t> inputShape = {1, 4, 20, 20},
                                                             ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32) {
     auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
+    params.front()->set_friendly_name("Param_1");
     auto split = ngraph::builder::makeSplit(params[0], ngPrc, 2, 1);
 
     auto conv1 = ngraph::builder::makeConvolution(split->output(0), ngPrc, {3, 3}, {1, 1}, {0, 0}, {0, 0}, {1, 1},

inference-engine/tests/unit/inference_engine/ie_blob_test.cpp

@@ -325,6 +325,23 @@ TEST_F(BlobTests, canCreateBlobOnExistedMemory) {
 }
 
 
+// SetShape
+TEST_F(BlobTests, canSetShape) {
+    auto b = InferenceEngine::make_shared_blob<float>(
+            InferenceEngine::TensorDesc(InferenceEngine::Precision::FP32, {1, 2, 3}, InferenceEngine::ANY));
+    b->allocate();
+
+    ASSERT_NO_THROW(b->setShape({4, 5, 6}));
+
+    auto newDims = b->getTensorDesc().getDims();
+    ASSERT_EQ(newDims.size(), 3);
+    ASSERT_EQ(newDims[0], 4);
+    ASSERT_EQ(newDims[1], 5);
+    ASSERT_EQ(newDims[2], 6);
+}
+
+
+
 TEST_F(BlobTests, canModifyDataInRangedFor) {
     InferenceEngine::SizeVector v = {1, 2, 3};
     InferenceEngine::TBlob<int> blob({ InferenceEngine::Precision::I32, v, InferenceEngine::CHW });