diff --git a/src/plugins/intel_cpu/src/cpu_shape.h b/src/plugins/intel_cpu/src/cpu_shape.h index 0d6cc341443..adc2d5c9711 100644 --- a/src/plugins/intel_cpu/src/cpu_shape.h +++ b/src/plugins/intel_cpu/src/cpu_shape.h @@ -39,13 +39,21 @@ public: hasZeroDimensions = std::any_of(dims.begin(), dims.end(), [](size_t dim) { return dim == 0; } ); } - Shape(const VectorDims& minVals, const VectorDims& maxVals) { - minDims = minVals; - maxDims = maxVals; - type = minVals == maxVals ? ShapeType::Static : ShapeType::Dynamic; + Shape(const VectorDims& minDims, const VectorDims& maxDims) { + if (minDims.size() != maxDims.size()) { + IE_THROW() << "Can't create shape due to min/max vectors dims size mismatch"; + } + this->minDims = minDims; + this->maxDims = maxDims; initDims(); + if (std::any_of(dims.begin(), dims.end(), [](size_t dim) { return dim == Shape::UNDEFINED_DIM; } )) { + type = ShapeType::Dynamic; + } else { + type = ShapeType::Static; + } + hasZeroDimensions = std::any_of(dims.begin(), dims.end(), [](size_t dim) { return dim == 0; } ); } diff --git a/src/plugins/intel_cpu/src/mkldnn_graph_optimizer.cpp b/src/plugins/intel_cpu/src/mkldnn_graph_optimizer.cpp index 568c61d7f9f..354c91a20d9 100644 --- a/src/plugins/intel_cpu/src/mkldnn_graph_optimizer.cpp +++ b/src/plugins/intel_cpu/src/mkldnn_graph_optimizer.cpp @@ -269,9 +269,11 @@ void MKLDNNGraphOptimizer::FuseConvolutionMatMulAndBias(MKLDNNGraph &graph) { graphEdges.push_back(newEdge); parent->addEdge(newEdge); - auto partialShape = { parentEltwise->outputShapes[0].toPartialShape()[parentEltwise->getFusingAxis()] }; - parent->outputShapes[inNum] = Shape(partialShape); - parentEltwise->inputShapes.push_back(parent->outputShapes[0]); + const auto fusingAxis = parentEltwise->getFusingAxis(); + const auto& outShape = parentEltwise->getOutputShapeAtPort(0); + + parent->outputShapes[inNum] = Shape({outShape.getMinDims()[fusingAxis]}, {outShape.getMaxDims()[fusingAxis]}); + parentEltwise->inputShapes.push_back(parent->getOutputShapeAtPort(0)); } } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp index 9702ebf0d89..5f08b824685 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp @@ -37,11 +37,6 @@ inline void ConvertToCPUSpecificOpset(std::shared_ptr &nGraphF manager.register_pass(); manager.register_pass(precisions_array {{ ngraph::element::i64, ngraph::element::i32 }}); - // TODO: remove after dynamic shapes support in FullyConnectedNode - manager.get_pass_config()->set_callback([](const std::shared_ptr& node) -> bool { - return node->get_input_partial_shape(0).is_dynamic(); - }); - manager.run_passes(nGraphFunc); } diff --git a/src/plugins/intel_cpu/src/nodes/mkldnn_eltwise_node.cpp b/src/plugins/intel_cpu/src/nodes/mkldnn_eltwise_node.cpp index a5fc77dd4cf..65cb07f1ec9 100644 --- a/src/plugins/intel_cpu/src/nodes/mkldnn_eltwise_node.cpp +++ b/src/plugins/intel_cpu/src/nodes/mkldnn_eltwise_node.cpp @@ -1995,8 +1995,6 @@ void MKLDNNEltwiseNode::execute(mkldnn::stream strm) { dims_out[batchDimIdx] = static_cast(batchToProcess()); } - std::vector vecPostOpData(fqDataPtrs.size()); - args_ptrs.post_op_data = fqDataPtrs.data(); execPtr->exec(args_ptrs, dims_out); diff --git a/src/plugins/intel_cpu/src/nodes/mkldnn_fullyconnected_node.cpp b/src/plugins/intel_cpu/src/nodes/mkldnn_fullyconnected_node.cpp index 759507a2a6f..c2fb6acf419 100644 --- a/src/plugins/intel_cpu/src/nodes/mkldnn_fullyconnected_node.cpp +++ b/src/plugins/intel_cpu/src/nodes/mkldnn_fullyconnected_node.cpp @@ -15,18 +15,66 @@ #include #include "memory_desc/dnnl_blocked_memory_desc.h" #include "utils/cpu_utils.hpp" +#include using namespace mkldnn; using namespace MKLDNNPlugin; using namespace InferenceEngine; +namespace { + +struct FCKey { + DnnlMemoryDescCPtr inp0; + DnnlMemoryDescCPtr inp1; + DnnlMemoryDescCPtr bias; + DnnlMemoryDescCPtr out; + mkldnn::primitive_attr attr; + impl_desc_type implType; + + size_t hash() const; + bool operator==(const FCKey& rhs) const; +}; + +size_t FCKey::hash() const { + using namespace dnnl::impl; + using namespace dnnl::impl::primitive_hashing; + + size_t seed = 0; + + for (const auto& ptr : {inp0, inp1, bias, out}) { + if (ptr) { + seed = hash_combine(seed, get_md_hash(ptr->getDnnlDesc().data)); + } + } + + seed = hash_combine(seed, get_attr_hash(*attr.get())); + seed = hash_combine(seed, implType); + return seed; +} + +bool FCKey::operator==(const FCKey &rhs) const { + bool retVal = true; + if (inp0 != rhs.inp0) { + retVal = retVal && inp0 && rhs.inp0 && inp0->getDnnlDesc() == rhs.inp0->getDnnlDesc(); + } + if (inp1 != rhs.inp1) { + retVal = retVal && inp1 && rhs.inp1 && inp1->getDnnlDesc() == rhs.inp1->getDnnlDesc(); + } + if (bias != rhs.bias) { + retVal = retVal && bias && rhs.bias && bias->getDnnlDesc() == rhs.bias->getDnnlDesc(); + } + if (out != rhs.out) { + retVal = retVal && out && rhs.out && out->getDnnlDesc() == rhs.out->getDnnlDesc(); + } + retVal = retVal && *attr.get() == *rhs.attr.get() && + implType == rhs.implType; + return retVal; +} + +} // namespace + bool MKLDNNFullyConnectedNode::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { - if (isDynamicNgraphNode(op)) { - errorMessage = "Doesn't support op with dynamic shapes"; - return false; - } - const auto fc = std::dynamic_pointer_cast(op); if (!fc) { errorMessage = "Only legacy FullyConnected operation is supported"; @@ -36,8 +84,15 @@ bool MKLDNNFullyConnectedNode::isSupportedOperation(const std::shared_ptrget_input_shape(DATA_ID).size(), 2, 3, 4)) { - errorMessage = "Doesn't support 'data' input with rank: " + std::to_string(fc->get_input_shape(DATA_ID).size()); + const auto inRank = fc->get_input_partial_shape(DATA_ID).size(); + const auto weightRank = fc->get_input_partial_shape(WEIGHTS_ID).size(); + if (!one_of(inRank, 2, 3, 4)) { + errorMessage = "Doesn't support 'data' input with rank: " + std::to_string(inRank); + return false; + } + if ((one_of(inRank, 2, 3) && weightRank != 2) || (inRank == 4 && weightRank != 4)) { + errorMessage = "Doesn't support 'data' input with rank: " + std::to_string(inRank) + + " and 'weight' input with rank: " + std::to_string(weightRank); return false; } } catch (...) { @@ -52,7 +107,7 @@ MKLDNNFullyConnectedNode::MKLDNNFullyConnectedNode(const std::shared_ptrget_input_size() == 3; + withBiases = inputShapes.size() == 3; } else { IE_THROW(NotImplemented) << errorMessage; } @@ -108,17 +163,32 @@ void MKLDNNFullyConnectedNode::getSupportedDescriptors() { outputDataType = memory::data_type::bf16; } - const auto& inDims = getInputShapeAtPort(0).getStaticDims(); - const auto& outDims = getOutputShapeAtPort(0).getStaticDims(); + const auto& inShape = getInputShapeAtPort(DATA_ID); + inDims = inShape.getDims(); + const auto& weightDims = getInputShapeAtPort(WEIGHTS_ID).getStaticDims(); + outDims = getOutputShapeAtPort(0).getDims(); - if (inDims.size() == 3) { - weightsDims = InferenceEngine::SizeVector({static_cast(outDims[2]), static_cast(inDims[2])}); - } else { - weightsDims.push_back(outDims[1]); - for (int i = 1; i < inDims.size(); i++) - weightsDims.push_back(inDims[i]); + if (isDynamicNode()) { + auto inMinDims = inShape.getMinDims(); + auto inMaxDims = inShape.getMaxDims(); + + if (inMinDims.size() == 3) { + inMinDims.back() = weightDims.back(); + inMaxDims.back() = weightDims.back(); + } else { + for (size_t i = 1; i < inMinDims.size(); i++) { + inMinDims[i] = weightDims[i]; + inMaxDims[i] = weightDims[i]; + } + } + inDims = MemoryDescUtils::makeDummyShape(Shape(inMinDims, inMaxDims)).getStaticDims(); + + std::vector inShapes = {Shape(inDims), Shape(weightDims)}; + if (inputShapes.size() > 2) { + inShapes.emplace_back(getInputShapeAtPort(BIAS_ID)); + } + outDims = shapeInferGeneric(inShapes).front(); } - biasesDims.push_back(weightsDims[0]); for (auto format : getAvailableFormatsForDims(getInputShapeAtPort(0))) { auto in_candidate = mkldnn::memory::desc(MKLDNNExtensionUtils::convertToDnnlDims(inDims), inputDataType, format); @@ -128,56 +198,169 @@ void MKLDNNFullyConnectedNode::getSupportedDescriptors() { } } -void MKLDNNFullyConnectedNode::createPrimitive() { - if (prim) - return; +void MKLDNNFullyConnectedNode::prepareParams() { + auto srcMemPtr = getParentEdgesAtPort(0)[0]->getMemoryPtr(); + auto wghMemPtr = getParentEdgesAtPort(1)[0]->getMemoryPtr(); + auto dstMemPtr = getChildEdgesAtPort(0)[0]->getMemoryPtr(); + if (!dstMemPtr || !dstMemPtr->GetPrimitivePtr()) + IE_THROW() << "Destination memory hasn't been allocated."; + if (!srcMemPtr || !srcMemPtr->GetPrimitivePtr()) + IE_THROW() << "Input memory hasn't been allocated."; + if (!wghMemPtr || !wghMemPtr->GetPrimitivePtr()) + IE_THROW() << "Weight memory hasn't been allocated."; + MKLDNNMemoryPtr biasMemPtr = nullptr; + if (withBiases) { + biasMemPtr = getParentEdgesAtPort(2)[0]->getMemoryPtr(); + if (!biasMemPtr || !biasMemPtr->GetPrimitivePtr()) + IE_THROW() << "Input memory hasn't been allocated."; + } + + const NodeDesc *selected_pd = getSelectedPrimitiveDescriptor(); + if (selected_pd == nullptr) + IE_THROW() << "Preferable primitive descriptor is not set for node " << getName() << "."; + + auto initPrimitiveAttr = [&]() { + mkldnn::primitive_attr attr; + setPostOps(attr, dstMemPtr->getStaticDims()); + + return std::make_shared(std::move(attr)); + }; AttrPtr attr = initPrimitiveAttr(); - std::shared_ptr prim_desc; - prim_desc = std::make_shared( - createPrimitiveDescriptor(*attr)); - prim.reset(new inner_product_forward(*prim_desc)); + DnnlMemoryDescCPtr weightDesc = wghMemPtr->GetDescWithType(); + DnnlMemoryDescCPtr biasDesc = nullptr; + if (biasMemPtr) { + biasDesc = biasMemPtr->GetDescWithType(); + } - auto src = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive(); - auto dst = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive(); - if (withBiases) - primArgs = {{DNNL_ARG_SRC, src}, {DNNL_ARG_WEIGHTS, getParentEdgeAt(WEIGHTS_ID)->getMemory().GetPrimitive()}, - {DNNL_ARG_BIAS, getParentEdgeAt(BIAS_ID)->getMemory().GetPrimitive()}, {DNNL_ARG_DST, dst}}; - else - primArgs = {{DNNL_ARG_SRC, src}, {DNNL_ARG_WEIGHTS, getParentEdgeAt(WEIGHTS_ID)->getMemory().GetPrimitive()}, {DNNL_ARG_DST, dst}}; + DnnlMemoryDescCPtr inDesc = srcMemPtr->GetDescWithType(); + DnnlMemoryDescCPtr outDesc = dstMemPtr->GetDescWithType(); + + FCKey key = {inDesc, + weightDesc, + biasDesc, + outDesc, + *attr, + selected_pd->getImplementationType()}; + + auto engine = getEngine(); + + auto builder = [&engine](const FCKey& key) -> std::shared_ptr { + auto inDesc = key.inp0->getDnnlDesc(); + if (inDesc.dims().size() == 3) { + auto inDims = inDesc.dims(); + auto normalizedInDims = {inDims[0] * inDims[1], inDims[2]}; + inDesc = inDesc.reshape(normalizedInDims); + } + + auto outDesc = key.out->getDnnlDesc(); + if (outDesc.dims().size() == 3) { + auto outDims = outDesc.dims(); + auto normalizedOutDims = { outDims[0] * outDims[1], outDims[2] }; + outDesc = outDesc.reshape(normalizedOutDims); + } + + std::shared_ptr fcDsc; + if (key.bias) { + fcDsc = std::make_shared(mkldnn::prop_kind::forward_scoring, + inDesc, + key.inp1->getDnnlDesc(), + key.bias->getDnnlDesc(), + outDesc); + } else { + fcDsc = std::make_shared(mkldnn::prop_kind::forward_scoring, + inDesc, + key.inp1->getDnnlDesc(), + outDesc); + } + MKLDNNDescriptor desc(fcDsc); + primitive_desc_iterator itpd = desc.createPrimitiveDescriptorIterator(engine, key.attr); + inner_product_forward::primitive_desc prim_desc; + + while (static_cast(itpd)) { + impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); + + if (impl_type == key.implType) { + prim_desc = itpd.get(); + break; + } + if (!itpd.next_impl()) { + return nullptr; + } + } + + return std::make_shared(prim_desc); + }; + + auto cache = getRuntimeCache(); + auto result = cache->getOrCreate(key, builder); + + if (!result.first) { + IE_THROW() << "Primitive descriptor was not found for node " << getName() << "."; + } + + prim = result.first; + + primArgs[DNNL_ARG_SRC] = srcMemPtr->GetPrimitive(); + primArgs[DNNL_ARG_WEIGHTS] = wghMemPtr->GetPrimitive(); + primArgs[DNNL_ARG_DST] = dstMemPtr->GetPrimitive(); + + if (withBiases) { + primArgs[DNNL_ARG_BIAS] = biasMemPtr->GetPrimitive(); + } appendPostOpArgs(*attr, primArgs, binaryPostOpsArgs); + + auto reshapeMemory = [this](int argType) { + auto param = primArgs.find(argType); + if (param != primArgs.end()) { + auto oldMem = param->second; + auto dims = oldMem.get_desc().dims(); + if (dims.size() == 3) { + std::vector normalizedDims({dims[0] * dims[1], dims[2]}); + mkldnn::memory::desc newMemDesc(oldMem.get_desc().reshape(normalizedDims)); + mkldnn::memory newMem(newMemDesc, oldMem.get_engine(), oldMem.get_data_handle()); + primArgs.at(argType) = newMem; + } + } + }; + reshapeMemory(DNNL_ARG_SRC); + reshapeMemory(DNNL_ARG_DST); } void MKLDNNFullyConnectedNode::execute(mkldnn::stream strm) { if (prim) { - auto reshapeMemory = [this](int argType) { + // in cases parameter -> FullyConnected or dynamic shapes + // we keep old pointer to data in primArgs on second iteration with same input shapes + auto updateMemoryPtr = [this](int argType) { auto param = primArgs.find(argType); if (param != primArgs.end()) { - auto oldMem = param->second; - auto dims = oldMem.get_desc().dims(); - if (dims.size() == 3) { - std::vector normalizedDims({dims[0] * dims[1], dims[2]}); - mkldnn::memory::desc newMemDesc(oldMem.get_desc().reshape(normalizedDims)); - mkldnn::memory newMem(newMemDesc, oldMem.get_engine(), oldMem.get_data_handle()); - primArgs.at(argType) = newMem; + if (argType == DNNL_ARG_SRC && getInputShapeAtPort(DATA_ID).getRank() == 3) { + primArgs.at(argType).set_data_handle(getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetData()); + } + if (argType == DNNL_ARG_DST && getOutputShapeAtPort(0).getRank() == 3) { + primArgs.at(argType).set_data_handle(getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetData()); } } }; - reshapeMemory(DNNL_ARG_SRC); - reshapeMemory(DNNL_ARG_DST); + updateMemoryPtr(DNNL_ARG_SRC); + updateMemoryPtr(DNNL_ARG_DST); (*prim).execute(strm, primArgs); } } +void MKLDNNFullyConnectedNode::executeDynamicImpl(mkldnn::stream strm) { + execute(strm); +} + bool MKLDNNFullyConnectedNode::canFuse(const MKLDNNNodePtr& node) const { return canFuseSimpleOperation(node); } -void MKLDNNFullyConnectedNode::setPostOps(mkldnn::primitive_attr &attr, bool initWeights = false) { +void MKLDNNFullyConnectedNode::setPostOps(mkldnn::primitive_attr &attr, const VectorDims &dims, bool initWeights) { mkldnn::post_ops ops; auto getBinPostOpShape = [&](){ @@ -185,7 +368,7 @@ void MKLDNNFullyConnectedNode::setPostOps(mkldnn::primitive_attr &attr, bool ini VectorDims binaryShape(binaryShapeRank, 1); const size_t channelAxis = getFusingAxis(); // always use 1 as channelAxis for binary Shape, since oneDNN primitive is actually always 2D - binaryShape[1] = getOutputShapeAtPort(0).getStaticDims()[channelAxis]; + binaryShape[1] = dims[channelAxis]; return binaryShape; }; @@ -197,9 +380,8 @@ void MKLDNNFullyConnectedNode::setPostOps(mkldnn::primitive_attr &attr, bool ini } if (auto* eltwiseNode = dynamic_cast(node.get())) { - // TODO [DS]: change to shape from memory if (eltwiseNode->getMKLDNNAlgorithm() != mkldnn::algorithm::undef) { - eltwiseNode->appendPostOps(ops, getOutputShapeAtPort(0).getStaticDims()); + eltwiseNode->appendPostOps(ops, dims); } else { eltwiseNode->appendBinPostOps(ops, getBinPostOpShape(), binaryPostOpsArgs); } @@ -246,8 +428,8 @@ const std::vector& MKLDNNFullyConnectedNode::getPrimitivesPriori }; // WA: brgemm kernel contains bug that may lead to segfault in case of added post-ops and unaligned number of channels - size_t simdWidth = 16; - auto inputDims = inputShapes[0].getDims(); + const size_t simdWidth = 16; + auto inputDims = getInputShapeAtPort(DATA_ID).getDims(); if (inputDims.back() != Shape::UNDEFINED_DIM && inputDims.back() % simdWidth == 0) { priorities.insert(priorities.begin() + 1, impl_desc_type::brgemm_avx512_amx); priorities.insert(priorities.begin() + 2, impl_desc_type::brgemm_avx512); @@ -263,7 +445,7 @@ const std::vector& MKLDNNFullyConnectedNode::getPrimitivesPriori MKLDNNNode::AttrPtr MKLDNNFullyConnectedNode::initPrimitiveAttr() { auto attr = std::make_shared(mkldnn::primitive_attr()); - setPostOps(*attr); + setPostOps(*attr, outDims); return attr; } @@ -300,10 +482,11 @@ void MKLDNNFullyConnectedNode::createDescriptorInternal(const mkldnn::memory::de MKLDNNExtensionUtils::GetPlainFormatByRank(normalizedOutDims.size())); } - mkldnn::memory::desc wgh_candidate(MKLDNNExtensionUtils::convertToDnnlDims(weightsDims), wdt, mkldnn::memory::format_tag::any); + mkldnn::memory::desc wgh_candidate(MKLDNNExtensionUtils::convertToDnnlDims(getInputShapeAtPort(WEIGHTS_ID).getStaticDims()), + wdt, mkldnn::memory::format_tag::any); if (withBiases) { - mkldnn::memory::desc bias_candidate(MKLDNNExtensionUtils::convertToDnnlDims(inputShapes[BIAS_ID].getStaticDims()), bdt, + mkldnn::memory::desc bias_candidate(MKLDNNExtensionUtils::convertToDnnlDims(getInputShapeAtPort(BIAS_ID).getStaticDims()), bdt, mkldnn::memory::format_tag::any); MKLDNNDescriptor desc(std::shared_ptr( new inner_product_forward::desc(prop_kind::forward_scoring, in_candidate, wgh_candidate, @@ -319,8 +502,21 @@ void MKLDNNFullyConnectedNode::createDescriptorInternal(const mkldnn::memory::de void MKLDNNFullyConnectedNode::createDescriptor(const std::vector &inputDesc, const std::vector &outputDesc) { - createDescriptorInternal(MemoryDescUtils::convertToDnnlMemoryDesc(inputDesc[0])->getDnnlDesc(), - MemoryDescUtils::convertToDnnlMemoryDesc(outputDesc[0])->getDnnlDesc()); + MemoryDescPtr inpDesc; + if (inputDesc[0]->isDefined()) { + inpDesc = inputDesc[0]; + } else { + inpDesc = inputDesc[0]->cloneWithNewDims(inDims); + } + + MemoryDescPtr outDesc; + if (outputDesc[0]->isDefined()) { + outDesc = outputDesc[0]; + } else { + outDesc = outputDesc[0]->cloneWithNewDims(outDims); + } + createDescriptorInternal(MemoryDescUtils::convertToDnnlMemoryDesc(inpDesc)->getDnnlDesc(), + MemoryDescUtils::convertToDnnlMemoryDesc(outDesc)->getDnnlDesc()); } std::shared_ptr MKLDNNFullyConnectedNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) { @@ -330,6 +526,11 @@ std::shared_ptr MKLDNNFullyConnectedNode::getSrcMemDesc(mkldnn::prim return std::make_shared(MKLDNNExtensionUtils::DataTypeToIEPrecision( static_cast(desc.data.data_type)), getInputShapeAtPort(idx)); } + + if (getInputShapeAtPort(idx).isDynamic()) { + return MKLDNNExtensionUtils::makeUndefinedDesc(desc, getInputShapeAtPort(idx)); + } + return MKLDNNExtensionUtils::makeDescriptor(desc); } @@ -340,6 +541,11 @@ std::shared_ptr MKLDNNFullyConnectedNode::getDstMemDesc(mkldnn::prim return std::make_shared(MKLDNNExtensionUtils::DataTypeToIEPrecision( static_cast(desc.data.data_type)), getOutputShapeAtPort(idx)); } + + if (getOutputShapeAtPort(idx).isDynamic()) { + return MKLDNNExtensionUtils::makeUndefinedDesc(desc, getOutputShapeAtPort(idx)); + } + return MKLDNNExtensionUtils::makeDescriptor(desc); } diff --git a/src/plugins/intel_cpu/src/nodes/mkldnn_fullyconnected_node.h b/src/plugins/intel_cpu/src/nodes/mkldnn_fullyconnected_node.h index c8394bb1afd..a42442f7543 100644 --- a/src/plugins/intel_cpu/src/nodes/mkldnn_fullyconnected_node.h +++ b/src/plugins/intel_cpu/src/nodes/mkldnn_fullyconnected_node.h @@ -18,7 +18,6 @@ public: std::vector getAvailableFormatsForDims(const Shape &dims) const override; void getSupportedDescriptors() override; - void createPrimitive() override; void execute(mkldnn::stream strm) override; bool created() const override; @@ -49,15 +48,17 @@ public: std::shared_ptr initPrimitiveAttr() override; + void prepareParams() override; + void executeDynamicImpl(mkldnn::stream strm) override; + private: void createDescriptorInternal(const mkldnn::memory::desc &inputDesc, const mkldnn::memory::desc &outputDesc); - InferenceEngine::SizeVector weightsDims; - InferenceEngine::SizeVector biasesDims; + VectorDims inDims; + VectorDims outDims; - std::vector PostOpsIntBlobMemory; - void setPostOps(mkldnn::primitive_attr &attr, bool initWeights); + void setPostOps(mkldnn::primitive_attr &attr, const VectorDims &dims, bool initWeights = false); bool withBiases = false; diff --git a/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.cpp b/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.cpp index 98889274ff7..d8430b095b9 100644 --- a/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.cpp +++ b/src/plugins/intel_cpu/src/nodes/mkldnn_matmul_node.cpp @@ -155,7 +155,6 @@ void MKLDNNMatMulNode::setPostOps(mkldnn::primitive_attr &attr, const VectorDims for (const auto &node : fusedWith) { if (auto* eltwiseNode = dynamic_cast(node.get())) { - // TODO [DS]: change to shape from memory if (eltwiseNode->getMKLDNNAlgorithm() != mkldnn::algorithm::undef) { eltwiseNode->appendPostOps(ops, dims); } else { diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/mat_mul.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/mat_mul.cpp index cad52b8b445..165342661ed 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/mat_mul.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/mat_mul.cpp @@ -210,37 +210,83 @@ const auto fusingBias = fusingSpecificParams{std::make_shared(std: /* ============= FullyConnected ============= */ namespace fullyConnected { -const std::vector IS2D = { - {static_shapes_to_test_representation({{59, 1}, {1, 120}}), {false, false}}, - {static_shapes_to_test_representation({{59, 1}, {1, 120}}), {true, false}}, +const std::vector IS2D_smoke = { {static_shapes_to_test_representation({{59, 1}, {1, 120}}), {false, true}}, {static_shapes_to_test_representation({{59, 1}, {1, 120}}), {true, true}}, {static_shapes_to_test_representation({{59, 120}, {120, 1}}), {false, false}}, - {static_shapes_to_test_representation({{59, 120}, {120, 1}}), {true, false}}, - {static_shapes_to_test_representation({{59, 120}, {120, 1}}), {false, true}}, {static_shapes_to_test_representation({{59, 120}, {120, 1}}), {true, true}}, {static_shapes_to_test_representation({{1, 120}, {120, 59}}), {false, false}}, {static_shapes_to_test_representation({{1, 120}, {120, 59}}), {true, false}}, - {static_shapes_to_test_representation({{1, 120}, {120, 59}}), {false, true}}, - {static_shapes_to_test_representation({{1, 120}, {120, 59}}), {true, true}}, - {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {false, false}}, {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {true, false}}, {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {false, true}}, - {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {true, true}}, + + { + { + {{-1, -1}, {{20, 60}, {20, 60}}}, + {{60, 120}, {{60, 120}, {60, 120}}} + }, + {false, false} + }, + { + { + {{{0, 100}, {0, 12}}, {{20, 1}, {14, 1}, {20, 1}, {14, 1}}}, + {{1, 120}, {{1, 120}, {1, 120}, {1, 120}, {1, 120}}} + }, + {true, true} + }, }; -std::vector fusingParamsSet2D { +const std::vector IS2D_nightly = { + {static_shapes_to_test_representation({{59, 1}, {1, 120}}), {false, false}}, + {static_shapes_to_test_representation({{59, 1}, {1, 120}}), {true, false}}, + + {static_shapes_to_test_representation({{59, 120}, {120, 1}}), {true, false}}, + {static_shapes_to_test_representation({{59, 120}, {120, 1}}), {false, true}}, + + {static_shapes_to_test_representation({{1, 120}, {120, 59}}), {true, true}}, + {static_shapes_to_test_representation({{1, 120}, {120, 59}}), {false, true}}, + + {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {true, true}}, + {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {false, false}}, + + { + { + {{-1, -1}, {{71, 128}, {50, 128}}}, + {{128, 20}, {{128, 20}, {128, 20}}} + }, + {false, false} + }, + { + { + {{-1, 59}, {{10, 59}, {15, 59}, {15, 59}}}, + {{59, 1}, {{59, 1}, {59, 1}, {59, 1}}} + }, + {true, false} + }, + { + { + {{{0, 120}, 59}, {{5, 59}, {11, 59}, {5, 59}, {10, 59}}}, + {{59, 120}, {{59, 120}, {59, 120}, {59, 120}, {59, 120}}} + }, + {false, true} + }, +}; + +std::vector fusingParamsSet2D_smoke { emptyFusingSpec, fusingBias, - fusingRelu, fusingMultiplyPerChannel, + fusingFakeQuantizePerTensorRelu, +}; + +std::vector fusingParamsSet2D_nightly { + fusingRelu, fusingScaleShift, // EltwiseMulAdd fusing fusingPReluPerTensor, fusingFakeQuantizePerChannelRelu, - fusingFakeQuantizePerTensorRelu, }; std::vector fusingParamsSet2DBF16 { @@ -250,7 +296,7 @@ std::vector fusingParamsSet2DBF16 { fusingPReluPerTensor, }; -const auto testParams2D = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D), +const auto testParams2D_smoke = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_smoke), ::testing::Values(ElementType::f32), ::testing::Values(ElementType::undefined), ::testing::Values(ElementType::undefined), @@ -258,10 +304,10 @@ const auto testParams2D = ::testing::Combine(::testing::Combine(::testing::Value ::testing::Values(CommonTestUtils::DEVICE_CPU), ::testing::Values(emptyAdditionalConfig)), ::testing::Values(MatMulNodeType::FullyConnected), - ::testing::ValuesIn(fusingParamsSet2D), + ::testing::ValuesIn(fusingParamsSet2D_smoke), ::testing::ValuesIn(filterSpecificParams())); -const auto testParams2DBF16 = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D), +const auto testParams2DBF16_smoke = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_smoke), ::testing::ValuesIn(netPRCs), ::testing::Values(ElementType::undefined), ::testing::Values(ElementType::undefined), @@ -272,28 +318,105 @@ const auto testParams2DBF16 = ::testing::Combine(::testing::Combine(::testing::V ::testing::ValuesIn(fusingParamsSet2DBF16), ::testing::ValuesIn(filterSpecificParams())); -INSTANTIATE_TEST_SUITE_P(smoke_FC_2D, MatMulLayerCPUTest, testParams2D, MatMulLayerCPUTest::getTestCaseName); -INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_BF16, MatMulLayerCPUTest, testParams2DBF16, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_FC_2D, MatMulLayerCPUTest, testParams2D_smoke, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_BF16, MatMulLayerCPUTest, testParams2DBF16_smoke, MatMulLayerCPUTest::getTestCaseName); -const std::vector IS3D = { +const auto testParams2D_nightly = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_nightly), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(helpers::InputLayerType::CONSTANT), + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::Values(emptyAdditionalConfig)), + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2D_nightly), + ::testing::ValuesIn(filterSpecificParams())); + +const auto testParams2DBF16_nightly = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_nightly), + ::testing::ValuesIn(netPRCs), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(helpers::InputLayerType::CONSTANT), + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::ValuesIn(additionalConfig)), + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2DBF16), + ::testing::ValuesIn(filterSpecificParams())); + +INSTANTIATE_TEST_SUITE_P(nightly_FC_2D, MatMulLayerCPUTest, testParams2D_nightly, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(nightly_FC_2D_BF16, MatMulLayerCPUTest, testParams2DBF16_nightly, MatMulLayerCPUTest::getTestCaseName); + +const std::vector IS3D_smoke = { {static_shapes_to_test_representation({{1, 32, 120}, {120, 5}}), {false, false}}, - {static_shapes_to_test_representation({{1, 32, 120}, {120, 5}}), {true, false}}, {static_shapes_to_test_representation({{1, 32, 120}, {120, 5}}), {false, true}}, + + {static_shapes_to_test_representation({{1, 32, 120}, {120, 50}}), {true, false}}, + {static_shapes_to_test_representation({{1, 32, 120}, {120, 50}}), {false, true}}, + + { + { + {{1, 5, 32}, {{1, 5, 32}, {1, 5, 32}}}, + {{32, 3}, {{32, 3}, {32, 3}}} + }, + {false, true} + }, + + {static_shapes_to_test_representation({{1, 429}, {1, 429, 1}}), {true, true}}, + { + { + {{-1, -1}, {{1, 129}, {2, 129}, {1, 129}, {2, 129}}}, + {{1, 129, 1}, {{1, 129, 1}, {1, 129, 1}, {1, 129, 1}, {1, 129, 1}}} + }, + {true, true} + }, + + { + { + {{{0, 60}, {0, 60}, {0, 60}}, {{1, 3, 14}, {1, 7, 14}}}, + {{14, 10}, {{14, 10}, {14, 10}}} + }, + {true, true} + }, +}; + +const std::vector IS3D_nightly = { + {static_shapes_to_test_representation({{1, 32, 120}, {120, 5}}), {true, false}}, {static_shapes_to_test_representation({{1, 32, 120}, {120, 5}}), {true, true}}, {static_shapes_to_test_representation({{1, 32, 120}, {120, 50}}), {false, false}}, - {static_shapes_to_test_representation({{1, 32, 120}, {120, 50}}), {true, false}}, - {static_shapes_to_test_representation({{1, 32, 120}, {120, 50}}), {false, true}}, {static_shapes_to_test_representation({{1, 32, 120}, {120, 50}}), {true, true}}, - {static_shapes_to_test_representation({{1, 429}, {1, 429, 1}}), {true, true}} + { + { + {{-1, -1, -1}, {{1, 32, 120}, {1, 12, 120}}}, + {{120, 3}, {{120, 3}, {120, 3}}} + }, + {false, false} + }, + { + { + {{-1, -1, 50}, {{1, 2, 50}, {1, 10, 50}, {1, 2, 50}, {2, 2, 50}}}, + {{50, 7}, {{50, 7}, {50, 7}, {50, 7}, {50, 7}}} + }, + {true, false} + }, + { + { + {{-1, -1, 32}, {{1, 5, 32}, {1, 5, 32}}}, + {{32, 3}, {{32, 3}, {32, 3}}} + }, + {false, true} + }, }; -std::vector fusingParamsSet3D { +std::vector fusingParamsSet3D_smoke { emptyFusingSpec, fusingBias, fusingMultiplyPerChannel, fusingFakeQuantizePerChannel, +}; + +std::vector fusingParamsSet3D_nightly { fusingFakeQuantizePerTensorRelu, }; @@ -303,7 +426,7 @@ std::vector fusingParamsSet3DBF16 { fusingMultiplyPerChannel, }; -const auto fullyConnectedParams3D = ::testing::Combine(::testing::ValuesIn(IS3D), +const auto fullyConnectedParams3D_smoke = ::testing::Combine(::testing::ValuesIn(IS3D_smoke), ::testing::Values(ElementType::f32), ::testing::Values(ElementType::undefined), ::testing::Values(ElementType::undefined), @@ -311,7 +434,7 @@ const auto fullyConnectedParams3D = ::testing::Combine(::testing::ValuesIn(IS3D) ::testing::Values(CommonTestUtils::DEVICE_CPU), ::testing::Values(emptyAdditionalConfig)); -const auto fullyConnectedParams3DBF16 = ::testing::Combine(::testing::ValuesIn(IS3D), +const auto fullyConnectedParams3DBF16_smoke = ::testing::Combine(::testing::ValuesIn(IS3D_smoke), ::testing::ValuesIn(netPRCs), ::testing::Values(ElementType::undefined), ::testing::Values(ElementType::undefined), @@ -319,18 +442,47 @@ const auto fullyConnectedParams3DBF16 = ::testing::Combine(::testing::ValuesIn(I ::testing::Values(CommonTestUtils::DEVICE_CPU), ::testing::ValuesIn(additionalConfig)); -const auto testParams3D = ::testing::Combine(fullyConnectedParams3D, +const auto testParams3D_smoke = ::testing::Combine(fullyConnectedParams3D_smoke, ::testing::Values(MatMulNodeType::FullyConnected), - ::testing::ValuesIn(fusingParamsSet3D), + ::testing::ValuesIn(fusingParamsSet3D_smoke), ::testing::ValuesIn(filterSpecificParams())); -const auto testParams3DBF16 = ::testing::Combine(fullyConnectedParams3DBF16, +const auto testParams3DBF16_smoke = ::testing::Combine(fullyConnectedParams3DBF16_smoke, ::testing::Values(MatMulNodeType::FullyConnected), ::testing::ValuesIn(fusingParamsSet3DBF16), ::testing::ValuesIn(filterSpecificParams())); -INSTANTIATE_TEST_SUITE_P(smoke_FC_3D, MatMulLayerCPUTest, testParams3D, MatMulLayerCPUTest::getTestCaseName); -INSTANTIATE_TEST_SUITE_P(smoke_FC_3D_BF16, MatMulLayerCPUTest, testParams3DBF16, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_FC_3D, MatMulLayerCPUTest, testParams3D_smoke, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_FC_3D_BF16, MatMulLayerCPUTest, testParams3DBF16_smoke, MatMulLayerCPUTest::getTestCaseName); + +const auto fullyConnectedParams3D_nightly = ::testing::Combine(::testing::ValuesIn(IS3D_nightly), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(helpers::InputLayerType::CONSTANT), + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::Values(emptyAdditionalConfig)); + +const auto fullyConnectedParams3DBF16_nightly = ::testing::Combine(::testing::ValuesIn(IS3D_nightly), + ::testing::ValuesIn(netPRCs), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(helpers::InputLayerType::CONSTANT), + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::ValuesIn(additionalConfig)); + +const auto testParams3D_nightly = ::testing::Combine(fullyConnectedParams3D_nightly, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet3D_nightly), + ::testing::ValuesIn(filterSpecificParams())); + +const auto testParams3DBF16_nightly = ::testing::Combine(fullyConnectedParams3DBF16_nightly, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet3DBF16), + ::testing::ValuesIn(filterSpecificParams())); + +INSTANTIATE_TEST_SUITE_P(nightly_FC_3D, MatMulLayerCPUTest, testParams3D_nightly, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(nightly_FC_3D_BF16, MatMulLayerCPUTest, testParams3DBF16_nightly, MatMulLayerCPUTest::getTestCaseName); std::vector> filterAdditionalConfig_Brgemm() { std::vector> additionalConfig = { @@ -343,16 +495,50 @@ std::vector> filterAdditionalConfig_Brgemm() return additionalConfig; } -const std::vector IS2D_Brgemm = { - {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {false, false}}, +const std::vector IS2D_Brgemm_smoke = { {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {true, false}}, - {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {false, true}}, {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {true, true}}, {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {false, false}}, - {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {true, false}}, {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {false, true}}, + + { + { + {{-1, -1}, {{12, 16}, {25, 16}, {12, 16}, {25, 16}}}, + {{16, 35}, {{16, 35}, {16, 35}, {16, 35}, {16, 35}}} + }, + {false, false} + }, + { + { + {{{0, 50}, {0, 50}}, {{17, 48}, {15, 48}}}, + {{48, 15}, {{48, 15}, {48, 15}}} + }, + {true, true} + }, +}; + +const std::vector IS2D_Brgemm_nightly = { + {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {false, false}}, + {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {false, true}}, + + {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {true, false}}, {static_shapes_to_test_representation({{71, 128}, {128, 20}}), {true, true}}, + + { + { + {{-1, 128}, {{11, 128}, {20, 128}, {11, 128}, {15, 128}}}, + {{128, 11}, {{128, 11}, {128, 11}, {128, 11}, {128, 11}}} + }, + {true, false} + }, + { + { + {{{0, 50}, 32}, {{50, 32}, {23, 32}}}, + {{32, 21}, {{32, 21}, {32, 21}}} + }, + {false, true} + }, }; std::vector filterSpecificParams_Brgemm() { @@ -364,7 +550,7 @@ std::vector filterSpecificParams_Brgemm() { return specificParams; } -const auto fullyConnectedParams2D_Brgemm = ::testing::Combine(::testing::ValuesIn(IS2D_Brgemm), +const auto fullyConnectedParams2D_Brgemm_smoke = ::testing::Combine(::testing::ValuesIn(IS2D_Brgemm_smoke), ::testing::Values(ElementType::f32), ::testing::Values(ElementType::undefined), ::testing::Values(ElementType::undefined), @@ -372,12 +558,27 @@ const auto fullyConnectedParams2D_Brgemm = ::testing::Combine(::testing::ValuesI ::testing::Values(CommonTestUtils::DEVICE_CPU), ::testing::ValuesIn(filterAdditionalConfig_Brgemm())); -const auto testParams2D_Brgemm = ::testing::Combine(fullyConnectedParams2D_Brgemm, +const auto testParams2D_Brgemm_smoke = ::testing::Combine(fullyConnectedParams2D_Brgemm_smoke, ::testing::Values(MatMulNodeType::FullyConnected), - ::testing::ValuesIn(fusingParamsSet2D), + ::testing::ValuesIn(fusingParamsSet2D_smoke), ::testing::ValuesIn(filterSpecificParams_Brgemm())); -INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_Brgemm, MatMulLayerCPUTest, testParams2D_Brgemm, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_Brgemm, MatMulLayerCPUTest, testParams2D_Brgemm_smoke, MatMulLayerCPUTest::getTestCaseName); + +const auto fullyConnectedParams2D_Brgemm_nightly = ::testing::Combine(::testing::ValuesIn(IS2D_Brgemm_nightly), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(helpers::InputLayerType::CONSTANT), + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::ValuesIn(filterAdditionalConfig_Brgemm())); + +const auto testParams2D_Brgemm_nightly = ::testing::Combine(fullyConnectedParams2D_Brgemm_nightly, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2D_nightly), + ::testing::ValuesIn(filterSpecificParams_Brgemm())); + +INSTANTIATE_TEST_SUITE_P(nightly_FC_2D_Brgemm, MatMulLayerCPUTest, testParams2D_Brgemm_nightly, MatMulLayerCPUTest::getTestCaseName); } // namespace fullyConnected