[CPU] Fix for Int8 models serialization issues (#7554)

inference-engine/src/mkldnn_plugin/mkldnn_extension.cpp

@@ -10,6 +10,8 @@
 
 #include <ngraph/ngraph.hpp>
 #include <ngraph_ops/type_relaxed.hpp>
+#include <ngraph_ops/nms_ie_internal.hpp>
+#include <ngraph_ops/nms_static_shape_ie.hpp>
 
 #include <mutex>
 
@@ -95,9 +97,22 @@ std::map<std::string, ngraph::OpSet> MKLDNNExtension::getOpSets() {
         return opset;
     };
 
+    auto ie_internal_opset = []() {
+        ngraph::OpSet opset;
+
+#define NGRAPH_OP(NAME, NAMESPACE) opset.insert<NAMESPACE::NAME>();
+        NGRAPH_OP(NonMaxSuppressionIEInternal, ngraph::op::internal)
+        NGRAPH_OP(NmsStaticShapeIE<ov::op::v8::MulticlassNms>, ngraph::op::internal)
+        NGRAPH_OP(NmsStaticShapeIE<ov::op::v8::MatrixNms>, ngraph::op::internal)
+#undef NGRAPH_OP
+
+        return opset;
+    };
+
     static std::map<std::string, ngraph::OpSet> opsets = {
         { "cpu_plugin_opset", cpu_plugin_opset() },
-        { "type_relaxed_opset", type_relaxed_opset() }
+        { "type_relaxed_opset", type_relaxed_opset() },
+        { "ie_internal_opset", ie_internal_opset() },
     };
 
     return opsets;

inference-engine/src/mkldnn_plugin/ngraph_transformations/op/fully_connected.cpp

@@ -11,7 +11,7 @@ MKLDNNPlugin::FullyConnectedNode::FullyConnectedNode(const ngraph::Output<Node>&
                                                      const ngraph::Shape& output_shape,
                                                      const ngraph::element::Type output_type)
     : Op({A, B}), m_output_shape(output_shape), m_output_type(output_type) {
-    constructor_validate_and_infer_types();
+    validate_and_infer_types();
 }
 
 MKLDNNPlugin::FullyConnectedNode::FullyConnectedNode(const ngraph::Output<Node>& A,
@@ -20,15 +20,15 @@ MKLDNNPlugin::FullyConnectedNode::FullyConnectedNode(const ngraph::Output<Node>&
                                                      const ngraph::Shape& output_shape,
                                                      const ngraph::element::Type output_type)
     : Op({A, B, C}), m_output_shape(output_shape), m_output_type(output_type) {
-    constructor_validate_and_infer_types();
+    validate_and_infer_types();
 }
 
 std::shared_ptr<ngraph::Node> MKLDNNPlugin::FullyConnectedNode::clone_with_new_inputs(const ngraph::OutputVector& new_args) const {
     check_new_args_count(this, new_args);
     if (new_args.size() == 2) {
-        return std::make_shared<MKLDNNPlugin::FullyConnectedNode>(new_args.at(0), new_args.at(1), m_output_shape);
+        return std::make_shared<MKLDNNPlugin::FullyConnectedNode>(new_args.at(0), new_args.at(1), m_output_shape, m_output_type);
     } else if (new_args.size() == 3) {
-        return std::make_shared<MKLDNNPlugin::FullyConnectedNode>(new_args.at(0), new_args.at(1), new_args.at(2), m_output_shape);
+        return std::make_shared<MKLDNNPlugin::FullyConnectedNode>(new_args.at(0), new_args.at(1), new_args.at(2), m_output_shape, m_output_type);
     }
 
     throw ngraph::ngraph_error("Unsupported number of arguments for FullyConnected operation");
@@ -42,5 +42,6 @@ void MKLDNNPlugin::FullyConnectedNode::validate_and_infer_types() {
 bool MKLDNNPlugin::FullyConnectedNode::visit_attributes(ngraph::AttributeVisitor &visitor) {
     visitor.on_attribute("out-size", m_output_size);
     visitor.on_attribute("out-shape", m_output_shape);
+    visitor.on_attribute("out-type", m_output_type);
     return true;
 }

inference-engine/src/mkldnn_plugin/ngraph_transformations/op/leaky_relu.cpp

@@ -10,7 +10,7 @@ MKLDNNPlugin::LeakyReluNode::LeakyReluNode(const ngraph::Output<ngraph::Node> &d
                                            const float &negative_slope,
                                            const ngraph::element::Type output_type)
     : Op({data}), m_negative_slope(negative_slope), m_output_type(output_type) {
-    constructor_validate_and_infer_types();
+    validate_and_infer_types();
 }
 
 std::shared_ptr<ngraph::Node> MKLDNNPlugin::LeakyReluNode::clone_with_new_inputs(const ngraph::OutputVector& new_args) const {
@@ -27,5 +27,6 @@ void MKLDNNPlugin::LeakyReluNode::validate_and_infer_types() {
 
 bool MKLDNNPlugin::LeakyReluNode::visit_attributes(ngraph::AttributeVisitor &visitor) {
     visitor.on_attribute("negative_slope", m_negative_slope);
+    visitor.on_attribute("out-type", m_output_type);
     return true;
 }

inference-engine/src/mkldnn_plugin/ngraph_transformations/op/power_static.cpp

@@ -12,7 +12,7 @@ MKLDNNPlugin::PowerStaticNode::PowerStaticNode(const ngraph::Output<Node> &data,
                                                const float &shift,
                                                const ngraph::element::Type output_type)
     : Op({data}), scale(scale), power(power), shift(shift), m_output_type(output_type) {
-    constructor_validate_and_infer_types();
+    validate_and_infer_types();
 }
 
 std::shared_ptr<ngraph::Node> MKLDNNPlugin::PowerStaticNode::clone_with_new_inputs(const ngraph::OutputVector &new_args) const {
@@ -31,5 +31,6 @@ bool MKLDNNPlugin::PowerStaticNode::visit_attributes(ngraph::AttributeVisitor &v
     visitor.on_attribute("scale", scale);
     visitor.on_attribute("power", power);
     visitor.on_attribute("shift", shift);
+    visitor.on_attribute("out-type", m_output_type);
     return true;
 }

inference-engine/src/mkldnn_plugin/ngraph_transformations/op/swish_cpu.cpp

@@ -8,7 +8,7 @@ constexpr ngraph::NodeTypeInfo MKLDNNPlugin::SwishNode::type_info;
 
 MKLDNNPlugin::SwishNode::SwishNode(const ngraph::Output<ngraph::Node> & input, const float alpha)
         : Op({input}), m_alpha(alpha) {
-    constructor_validate_and_infer_types();
+    validate_and_infer_types();
 }
 
 std::shared_ptr<ngraph::Node> MKLDNNPlugin::SwishNode::clone_with_new_inputs(const ngraph::OutputVector& new_args) const {

inference-engine/src/transformations/include/ngraph_ops/framework_node.hpp

@@ -56,6 +56,8 @@ public:
     OPENVINO_OP("FrameworkNode", "util");
     BWDCMP_RTTI_DECLARATION;
 
+    FrameworkNode() = default;
+
     explicit FrameworkNode(const OutputVector& inputs, size_t output_size = 1);
 
     void validate_and_infer_types() override;

inference-engine/src/transformations/include/ngraph_ops/nms_ie_internal.hpp

@@ -21,6 +21,8 @@ public:
     OPENVINO_OP("NonMaxSuppressionIEInternal", "util");
     BWDCMP_RTTI_DECLARATION;
 
+    NonMaxSuppressionIEInternal() = default;
+
     NonMaxSuppressionIEInternal(const Output<Node>& boxes,
                                 const Output<Node>& scores,
                                 const Output<Node>& max_output_boxes_per_class,

inference-engine/src/transformations/include/ngraph_ops/nms_static_shape_ie.hpp

@@ -22,6 +22,8 @@ class NmsStaticShapeIE : public BaseNmsOp {
 public:
     NGRAPH_RTTI_DECLARATION;
 
+    NmsStaticShapeIE() = default;
+
     using Attributes = typename BaseNmsOp::Attributes;
 
     /// \brief Constructs a NmsStaticShapeIE operation
@@ -38,6 +40,16 @@ public:
     std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override {
         return std::make_shared<NmsStaticShapeIE>(new_args.at(0), new_args.at(1), this->m_attrs);
     }
+
+private:
+    typedef struct {} init_rt_result;
+
+    init_rt_result init_rt_info() {
+        BaseNmsOp::get_rt_info()["opset"] = std::make_shared<ngraph::VariantWrapper<std::string>>("ie_internal_opset");
+        return {};
+    }
+
+    init_rt_result init_rt = init_rt_info();
 };
 
 template <typename BaseNmsOp>

inference-engine/tests/functional/plugin/cpu/shared_tests_instances/behavior/caching_tests.cpp

@@ -3,8 +3,12 @@
 //
 
 #include "behavior/caching/caching_tests.hpp"
+#include <ngraph_ops/nms_ie_internal.hpp>
+#include <ngraph_ops/framework_node.hpp>
+#include <ngraph_ops/nms_static_shape_ie.hpp>
 
 using namespace LayerTestsDefinitions;
+using namespace ngraph;
 
 namespace {
     static const std::vector<ngraph::element::Type> precisionsCPU = {
@@ -22,6 +26,54 @@ namespace {
             1, 2
     };
 
+    static const std::vector<ngraph::element::Type> precisionsCPUInternal = {
+            ngraph::element::f32
+    };
+
+    static const std::vector<std::size_t> batchSizesCPUInternal = {
+            1
+    };
+
+    static std::shared_ptr<ngraph::Function> simple_function_non_max_supression_internal(ngraph::element::Type, size_t) {
+        auto boxes = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1000, 4});
+        auto scores = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1, 1000});
+        auto max_output_boxes_per_class = opset1::Constant::create(element::i32, Shape{1}, {10});
+        auto iou_threshold = opset1::Constant::create(element::f32, Shape{1}, {0.75});
+        auto score_threshold = opset1::Constant::create(element::f32, Shape{1}, {0.7});
+        auto nms = std::make_shared<op::internal::NonMaxSuppressionIEInternal>(boxes, scores, max_output_boxes_per_class,
+                iou_threshold, score_threshold, 0, true, element::i32);
+        auto res = std::make_shared<ngraph::opset6::Result>(nms);
+        auto func = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
+        return func;
+    }
+
+    static std::shared_ptr<ngraph::Function> simple_function_matrix_nms_internal(ngraph::element::Type, size_t) {
+        auto boxes = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1000, 4});
+        auto scores = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1, 1000});
+        auto nms = std::make_shared<op::internal::NmsStaticShapeIE<ov::op::v8::MatrixNms>>(boxes, scores, ov::op::v8::MatrixNms::Attributes());
+        auto res = std::make_shared<ngraph::opset6::Result>(nms);
+        auto func = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
+        return func;
+    }
+
+    static std::shared_ptr<ngraph::Function> simple_function_multiclass_nms_internal(ngraph::element::Type, size_t) {
+        auto boxes = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1000, 4});
+        auto scores = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1, 1000});
+        auto nms = std::make_shared<op::internal::NmsStaticShapeIE<ov::op::v8::MulticlassNms>>(boxes, scores, ov::op::v8::MulticlassNms::Attributes());
+        auto res = std::make_shared<ngraph::opset6::Result>(nms);
+        auto func = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
+        return func;
+    }
+
+    static std::vector<nGraphFunctionWithName> internal_functions_cpu() {
+        std::vector<nGraphFunctionWithName> funcs = {
+            nGraphFunctionWithName { simple_function_non_max_supression_internal, "NonMaxSuppressionIEInternal"},
+            nGraphFunctionWithName { simple_function_matrix_nms_internal, "NmsStaticShapeIE_MatrixNms"},
+            nGraphFunctionWithName { simple_function_multiclass_nms_internal, "NmsStaticShapeIE_MulticlassNms"},
+        };
+        return funcs;
+    }
+
     INSTANTIATE_TEST_SUITE_P(smoke_CachingSupportCase_CPU, LoadNetworkCacheTestBase,
                             ::testing::Combine(
                                     ::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()),
@@ -29,4 +81,12 @@ namespace {
                                     ::testing::ValuesIn(batchSizesCPU),
                                     ::testing::Values(CommonTestUtils::DEVICE_CPU)),
                             LoadNetworkCacheTestBase::getTestCaseName);
+
+    INSTANTIATE_TEST_SUITE_P(smoke_CachingSupportCase_CPU_Internal, LoadNetworkCacheTestBase,
+                            ::testing::Combine(
+                                    ::testing::ValuesIn(internal_functions_cpu()),
+                                    ::testing::ValuesIn(precisionsCPUInternal),
+                                    ::testing::ValuesIn(batchSizesCPUInternal),
+                                    ::testing::Values(CommonTestUtils::DEVICE_CPU)),
+                            LoadNetworkCacheTestBase::getTestCaseName);
 } // namespace