diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp
index f072a3426d2..d6d1f483c85 100644
--- a/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp
+++ b/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp
@@ -132,6 +132,10 @@ void MKLDNNGraph::Replicate(const TensorIterator::Body &subgraph, const MKLDNNEx
 
         graphNodes.push_back(node);
         inputNodes[input->getName()] = node;
+
+        if (getInputTo(input).empty()) {
+            unused_data.insert(input);
+        }
     }
 
     // Step 2. Replicate all internal nodes.