From b5ea943267893200a3bff4bd8cf41715e142b906 Mon Sep 17 00:00:00 2001 From: Maxim Andronov Date: Fri, 28 Jan 2022 21:21:02 +0300 Subject: [PATCH] [CPU] MatMul dummy shapes creation fixed (#9858) --- .../src/memory_desc/cpu_memory_desc_utils.h | 6 +- .../intel_cpu/src/mkldnn_infer_request.cpp | 4 +- .../src/nodes/mkldnn_matmul_node.cpp | 81 +++++++++++- .../intel_cpu/src/nodes/mkldnn_matmul_node.h | 1 + .../plugin/cpu/single_layer_tests/mat_mul.cpp | 123 +++++++++++++++++- 5 files changed, 207 insertions(+), 8 deletions(-) diff --git a/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.h b/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.h index eb2b6a3a6ee..9a314b376cc 100644 --- a/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.h +++ b/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.h @@ -72,13 +72,15 @@ public: */ static InferenceEngine::TensorDesc convertToTensorDesc(const MemoryDesc& desc); + static constexpr Dim DEFAULT_DUMMY_VAL = 64; + /** * @brief Makes a dummy descriptor where all undefined values are replaced with the smallest value between the parameter and the upper bound dim * @param desc MemoryDesc from which the new descriptor is generated * @param dummyVal Dim value to replace undefined dimensions * @return a new MemoryDesc with dummy values instead of undefined dims */ - static std::shared_ptr makeDummyDesc(const MemoryDesc& desc, Dim dummyVal = 64); + static std::shared_ptr makeDummyDesc(const MemoryDesc& desc, Dim dummyVal = DEFAULT_DUMMY_VAL); /** * @brief Makes a static dummy shape where all undefined values are replaced with the smallest value between the parameter and the upper bound dim @@ -86,7 +88,7 @@ public: * @param dummyVal Dim value to replace undefined dimensions * @return a new Shape with dummy values instead of undefined dims */ - static Shape makeDummyShape(const Shape& shape, Dim dummyVal = 64); + static Shape makeDummyShape(const Shape& shape, Dim dummyVal = DEFAULT_DUMMY_VAL); /** * @brief Converts dim to string, undefined dim represented as ? diff --git a/src/plugins/intel_cpu/src/mkldnn_infer_request.cpp b/src/plugins/intel_cpu/src/mkldnn_infer_request.cpp index 710865fe23e..2a629812cca 100644 --- a/src/plugins/intel_cpu/src/mkldnn_infer_request.cpp +++ b/src/plugins/intel_cpu/src/mkldnn_infer_request.cpp @@ -717,7 +717,9 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In const auto shape = inputNodeItr->second->get_output_partial_shape(0); const bool isDynamic = shape.is_dynamic(); if (!shape.compatible(ov::PartialShape(data->getTensorDesc().getDims()))) { - IE_THROW() << "Can't SetBlob with name: " << name << ", because model input and blob are incompatible"; + IE_THROW() << "Can't SetBlob with name: " << name + << ", because model input (shape=" << shape + << ") and blob (shape=" << vec2str(data->getTensorDesc().getDims()) << ") are incompatible"; } if (!isDynamic && ngraph::shape_size(shape.to_shape()) != data->size()) { diff --git a/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.cpp b/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.cpp index 5fde4eb8914..2ad9394ce08 100644 --- a/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.cpp +++ b/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.cpp @@ -291,9 +291,10 @@ void MKLDNNMatMulNode::getSupportedDescriptors() { } } - std::vector staticInputShapes(2); - staticInputShapes[0] = inputShape0.isStatic() ? inputShape0 : MemoryDescUtils::makeDummyShape(inputShape0); - staticInputShapes[1] = inputShape1.isStatic() ? inputShape1 : MemoryDescUtils::makeDummyShape(inputShape1); + std::vector staticInputShapes{inputShape0, inputShape1}; + if (inputShape0.isDynamic() || inputShape1.isDynamic()) { + std::tie(staticInputShapes[0], staticInputShapes[1]) = makeDummyInputShapes(inputShape0, inputShape1); + } auto staticOutputShape = outputShape.isStatic() ? outputShape : Shape(shapeInferGeneric(staticInputShapes).front()); @@ -307,6 +308,80 @@ void MKLDNNMatMulNode::getSupportedDescriptors() { createDescriptor({inDataDesc[0], inDataDesc[1]}, {outDataDesc}); } +std::pair MKLDNNMatMulNode::makeDummyInputShapes(const Shape& in0, const Shape& in1) const { + if (in0.getRank() < 2 || in1.getRank() < 2) { + IE_THROW() << "Can't create dummy inputs with rank less 2"; + } + + if (in0.getRank() != in1.getRank()) { + IE_THROW() << "Can't create dummy inputs if input's rank not equal"; + } + + auto swapTranspDims = [&](VectorDims& in0, VectorDims& in1) { + if (transposeIn[0]) { + std::swap(in0[in0.size() - 1], in0[in0.size() - 2]); + } + if (transposeIn[1]) { + std::swap(in1[in1.size() - 1], in1[in1.size() - 2]); + } + }; + + auto inDims0 = in0.getDims(); + auto inDims1 = in1.getDims(); + + auto minDims0 = in0.getMinDims(); + auto maxDims0 = in0.getMaxDims(); + auto minDims1 = in1.getMinDims(); + auto maxDims1 = in1.getMaxDims(); + + swapTranspDims(inDims0, inDims1); + swapTranspDims(minDims0, minDims1); + swapTranspDims(maxDims0, maxDims1); + + auto fillDummy = [&](size_t idx0, size_t idx1) { + if (inDims0[idx0] == Shape::UNDEFINED_DIM && inDims1[idx1] == Shape::UNDEFINED_DIM) { + inDims0[idx0] = inDims1[idx1] = std::min(std::min(maxDims0[idx0], maxDims1[idx1]), + std::max(std::max(minDims0[idx0], minDims1[idx1]), static_cast(MemoryDescUtils::DEFAULT_DUMMY_VAL))); + } else { + if (inDims0[idx0] == Shape::UNDEFINED_DIM && inDims1[idx1] != Shape::UNDEFINED_DIM) { + if (inDims1[idx1] == 1 && minDims0[idx0] != Shape::UNDEFINED_DIM) { + inDims0[idx0] = std::max(minDims0[idx0], 1); + } else { + inDims0[idx0] = inDims1[idx1]; + } + } else if (inDims0[idx0] != Shape::UNDEFINED_DIM && inDims1[idx1] == Shape::UNDEFINED_DIM) { + if (inDims0[idx0] == 1 && minDims1[idx1] != Shape::UNDEFINED_DIM) { + inDims1[idx1] = std::max(minDims1[idx1], 1); + } else { + inDims1[idx1] = inDims0[idx0]; + } + } + } + }; + + // fill k + fillDummy(inDims0.size() - 1, inDims1.size() - 2); + + // fill m, n + if (inDims0[inDims0.size() - 2] == Shape::UNDEFINED_DIM) { + inDims0[inDims0.size() - 2] = std::min(maxDims0[inDims0.size() - 2], + std::max(minDims0[inDims0.size() - 2], static_cast(MemoryDescUtils::DEFAULT_DUMMY_VAL))); + } + if (inDims1[inDims1.size() - 1] == Shape::UNDEFINED_DIM) { + inDims1[inDims1.size() - 1] = std::min(maxDims1[inDims1.size() - 1], + std::max(minDims1[inDims1.size() - 1], static_cast(MemoryDescUtils::DEFAULT_DUMMY_VAL))); + } + + // fill batches + for (size_t i = 0; i < inDims0.size() - 2; i++) { + fillDummy(i, i); + } + + swapTranspDims(inDims0, inDims1); + + return {Shape(inDims0), Shape(inDims1)}; +} + void MKLDNNMatMulNode::createDescriptor(const std::vector& inputDesc, const std::vector& outputDesc) { std::shared_ptr matmul_desc; diff --git a/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.h b/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.h index 07cafc16dc6..d19c9ad67d8 100644 --- a/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.h +++ b/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.h @@ -47,6 +47,7 @@ protected: private: mkldnn::memory::desc getBiasDescFrom(const DnnlMemoryDescCPtr outMemDesc); + std::pair makeDummyInputShapes(const Shape& in0, const Shape& in1) const; bool withBiases; diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/mat_mul.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/mat_mul.cpp index c6b1c29c0ec..5e50425b2d4 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/mat_mul.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/mat_mul.cpp @@ -748,11 +748,116 @@ const std::vector IS_Dynamic = { }, { { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} - {{{1, 15}, {1, 15}, {1, 15}}, {{10, 10, 10}, {5, 5, 5}}}, // input 0 - {{{1, 15}, {1, 15}, {1, 15}}, {{10, 10, 10}, {5, 5, 5}}} // input 1 + {{ -1, 16 }, {{ 4, 16 }, { 2, 16 }}}, // input 0 + {{ {1, 5}, 12, -1, 4 }, {{ 1, 12, 16, 4 }, { 1, 12, 16, 4 }}} // input 1 }, {true, true} }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ -1, 12, -1, 16 }, {{ 1, 12, 4, 16 }, { 2, 12, 2, 16 }}}, // input 0 + {{ {1, 5}, 12, -1, 4 }, {{ 1, 12, 16, 4 }, { 1, 12, 16, 4 }}} // input 1 + }, + {false, false} + }, +}; + +const std::vector IS_Dynamic_nightly = { + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{{5, 15}, {1, 12}, {4, 15}}, {{10, 10, 10}, {5, 5, 5}}}, // input 0 + {{{1, 13}, {3, 15}, {1, 10}}, {{10, 10, 10}, {5, 5, 5}}} // input 1 + }, + {true, true} + }, + + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ {2, 10}, {3, 15}, -1, 16 }, {{ 2, 12, 4, 16 }, { 3, 12, 2, 16 }}}, // input 0 + {{ 1, 1, -1, 4 }, {{ 1, 1, 16, 4 }, { 1, 1, 16, 4 }}} // input 1 + }, + {true, true} + }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ 1, 1, -1, 16 }, {{ 1, 1, 4, 16 }, { 1, 1, 2, 16 }}}, // input 0 + {{ {2, 5}, {3, 15}, -1, 4 }, {{ 2, 12, 16, 4 }, { 2, 12, 16, 4 }}} // input 1 + }, + {false, false} + }, + + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ -1, 16 }, {{ 4, 16 }, { 2, 16 }}}, // input 0 + {{ {1, 5}, 12, -1, 4 }, {{ 1, 12, 16, 4 }, { 1, 12, 16, 4 }}} // input 1 + }, + {false, false} + }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ -1, {2, 15}, -1, 16 }, {{ 1, 12, 4, 16 }, { 2, 12, 2, 16 }}}, // input 0 + {{ -1, 4 }, {{ 16, 4 }, { 16, 4 }}} // input 1 + }, + {true, true} + }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ -1, {1, 15}, -1, 16 }, {{ 1, 12, 4, 16 }, { 2, 12, 2, 16 }}}, // input 0 + {{ -1, 4 }, {{ 16, 4 }, { 16, 4 }}} // input 1 + }, + {false, false} + }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ {1, 3}, {1, 9}, {1, 5}, {1, 10} }, {{ 1, 7, 4, 5 }, { 1, 7, 4, 4 }}}, // input 0 + {{ {1, 5}, {1, 7}, {1, 8}, {1, 5} }, {{ 1, 7, 5, 4 }, { 1, 7, 4, 4 }}} // input 1 + }, + {true, true} + }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ {1, 3}, {1, 9}, {1, 5}, {1, 10} }, {{ 1, 7, 4, 5 }, { 1, 7, 4, 4 }}}, // input 0 + {{ {1, 5}, {1, 7}, {1, 8}, {1, 5} }, {{ 1, 7, 5, 4 }, { 1, 7, 4, 4 }}} // input 1 + }, + {false, false} + }, + + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ 1, 7, 4, -1 }, {{ 1, 7, 4, 5 }, { 1, 7, 4, 4 }}}, // input 0 + {{ 1, 7, -1, 4 }, {{ 1, 7, 5, 4 }, { 1, 7, 4, 4 }}} // input 1 + }, + {true, true} + }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ 1, 7, 4, -1 }, {{ 1, 7, 4, 5 }, { 1, 7, 4, 4 }}}, // input 0 + {{ 1, 7, -1, 4 }, {{ 1, 7, 5, 4 }, { 1, 7, 4, 4 }}} // input 1 + }, + {false, false} + }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ -1, 12, -1, 16 }, {{ 1, 12, 4, 16 }, { 2, 12, 2, 16 }}}, // input 0 + {{ {1, 5}, 12, -1, 4 }, {{ 1, 12, 16, 4 }, { 1, 12, 16, 4 }}} // input 1 + }, + {true, true} + }, + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ -1, 12, -1, 16 }, {{ 1, 12, 4, 16 }, { 2, 12, 2, 16 }}}, // input 0 + {{ {1, 5}, 12, -1, 4 }, {{ 1, 12, 16, 4 }, { 1, 12, 16, 4 }}} // input 1 + }, + {true, false} + }, + + { + { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} + {{ -1, 12, -1, 16 }, {{ 1, 12, 4, 16 }, { 2, 12, 2, 16 }}}, // input 0 + {{ {1, 5}, 12, -1, 4 }, {{ 1, 12, 16, 4 }, { 1, 12, 16, 4 }}} // input 1 + }, + {false, true} + }, }; std::vector matmulFusingParams { @@ -801,6 +906,20 @@ const auto testParamsDynamic = ::testing::Combine(matMulParamsDynamic, INSTANTIATE_TEST_SUITE_P(smoke_MM_Dynamic, MatMulLayerCPUTest, testParamsDynamic, MatMulLayerCPUTest::getTestCaseName); +const auto matMulParamsDynamic_nightly = ::testing::Combine(::testing::ValuesIn(IS_Dynamic_nightly), + ::testing::ValuesIn(netPRCs), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(helpers::InputLayerType::PARAMETER), + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::ValuesIn(additionalConfig)); + +const auto testParamsDynamic_nightly = ::testing::Combine(matMulParamsDynamic_nightly, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::Values(emptyFusingSpec), + ::testing::ValuesIn(filterSpecificParams())); + +INSTANTIATE_TEST_SUITE_P(nightly_MM_Dynamic, MatMulLayerCPUTest, testParamsDynamic_nightly, MatMulLayerCPUTest::getTestCaseName); const std::vector IS_Dynamic_Fusing = { {