[CPU] Transpose constant folding on cpu plug-in side for MatMul op (#18877)
This commit is contained in:
Anton Voronov
+1
-1
@@ -17,7 +17,7 @@ namespace pass {
|
||||
|
||||
class TRANSFORMATIONS_API AlignEltwiseInputRanks : public MatcherPass {
|
||||
public:
|
||||
OPENVINO_RTTI("TRANSFORMATIONS_API", "0");
|
||||
OPENVINO_RTTI("AlignEltwiseInputRanks", "0");
|
||||
AlignEltwiseInputRanks();
|
||||
};
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ namespace pass {
|
||||
|
||||
class TRANSFORMATIONS_API MatMulConstTransposesExtraction : public MatcherPass {
|
||||
public:
|
||||
OPENVINO_RTTI("TRANSFORMATIONS_API", "0");
|
||||
OPENVINO_RTTI("MatMulConstTransposesExtraction", "0");
|
||||
MatMulConstTransposesExtraction();
|
||||
};
|
||||
|
||||
|
||||
@@ -95,6 +95,10 @@ void GraphOptimizer::ApplyCommonGraphOptimizations(Graph &graph) {
|
||||
FuseFCAndConvertOnWeights(graph);
|
||||
graph.RemoveDroppedNodes();
|
||||
|
||||
OV_ITT_SCOPE_NEXT(FIRST_INFERENCE, taskChain, "FuseFCAndTransposeOnWeights");
|
||||
FuseFCAndTransposeOnWeights(graph);
|
||||
graph.RemoveDroppedNodes();
|
||||
|
||||
OV_ITT_SCOPE_NEXT(FIRST_INFERENCE, taskChain, "FuseDeconvolutionAndSimpleOperation");
|
||||
FuseDeconvolutionAndSimpleOperation(graph);
|
||||
graph.RemoveDroppedNodes();
|
||||
@@ -805,10 +809,20 @@ void GraphOptimizer::FuseFCAndConvertOnWeights(Graph& graph) {
|
||||
// (e.g. fuse conversion with weights reordering)
|
||||
auto& graphNodes = graph.GetNodes();
|
||||
|
||||
auto isSuitablePattern = [](NodePtr parent) {
|
||||
bool res = true && parent->getType() == Type::Convert
|
||||
&& parent->getChildEdges().size() == 1
|
||||
&& parent->getChildEdgeAt(0)->getOutputNum() == 1
|
||||
&& parent->getChildEdgeAt(0)->getChild()->getType() == Type::FullyConnected
|
||||
&& one_of(parent->getOriginalInputPrecisionAtPort(0), Precision::FP16)
|
||||
&& one_of(parent->getOriginalOutputPrecisionAtPort(0), Precision::FP32, Precision::BF16)
|
||||
&& parent->isConstant();
|
||||
return res;
|
||||
};
|
||||
|
||||
for (auto parent : graphNodes) {
|
||||
if (parent->getType() == Type::Convert && parent->isConstant() && parent->getChildEdgeAt(0)->getChild()->getType() == Type::FullyConnected
|
||||
&& parent->getOriginalInputPrecisionAtPort(0) == Precision::FP16
|
||||
&& one_of(parent->getOriginalOutputPrecisionAtPort(0), Precision::FP32, Precision::BF16)) {
|
||||
if (isSuitablePattern(parent)) {
|
||||
CPU_GRAPH_OPTIMIZER_SCOPE(FuseFCAndConvertOnWeights);
|
||||
auto childNode = parent->getChildEdgeAt(0)->getChild();
|
||||
// set correct weight precision
|
||||
childNode->setOriginalInputPrecisionAtPort(1, parent->getOriginalInputPrecisionAtPort(0));
|
||||
@@ -817,6 +831,31 @@ void GraphOptimizer::FuseFCAndConvertOnWeights(Graph& graph) {
|
||||
}
|
||||
}
|
||||
|
||||
void GraphOptimizer::FuseFCAndTransposeOnWeights(Graph& graph) {
|
||||
// This optimization allows us to avoid transposing the weights in Transpose node and do it directly along with reordering in FC node
|
||||
auto& graphNodes = graph.GetNodes();
|
||||
|
||||
auto isSuitablePattern = [](NodePtr parent) {
|
||||
bool res = true && parent->getType() == Type::Transpose
|
||||
&& parent->getChildEdges().size() == 1
|
||||
&& parent->getChildEdgeAt(0)->getOutputNum() == 1
|
||||
&& parent->getChildEdgeAt(0)->getChild()->getType() == Type::FullyConnected
|
||||
&& parent->getOutputShapeAtPort(0).getRank() == 2
|
||||
&& parent->isConstant();
|
||||
return res;
|
||||
};
|
||||
|
||||
for (auto parent : graphNodes) {
|
||||
if (isSuitablePattern(parent)) {
|
||||
CPU_GRAPH_OPTIMIZER_SCOPE(FuseFCAndTransposeOnWeights);
|
||||
auto fcNode = std::dynamic_pointer_cast<FullyConnected>(parent->getChildEdgeAt(0)->getChild());
|
||||
fcNode->keepWeightsNonTransposed(true);
|
||||
auto transposeNode = std::dynamic_pointer_cast<Transpose>(parent);
|
||||
transposeNode->setOptimized(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void GraphOptimizer::FuseConvolutionAndZeroPoints(Graph &graph) {
|
||||
auto& graphNodes = graph.GetNodes();
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ private:
|
||||
void FuseMultiplyAndAdd(Graph &graph);
|
||||
void MergeConvertAndScaleShift(Graph& graph);
|
||||
void FuseFCAndConvertOnWeights(Graph& graph);
|
||||
void FuseFCAndTransposeOnWeights(Graph& graph);
|
||||
void FuseFullyConnectedAndSimpleOperation(Graph &graph);
|
||||
void FuseMatMulAndSimpleOperation(Graph &graph);
|
||||
void FuseConvolutionAndSimpleOperationThroughMaxPool(Graph &graph);
|
||||
|
||||
@@ -865,28 +865,30 @@ void Node::prepareMemory(dnnl::primitive_desc_iterator& itpd) {
|
||||
Node::prepareMemory(intDescs);
|
||||
}
|
||||
|
||||
MemoryPtr Node::prepareWeightMemory(DnnlMemoryDescPtr weightDesc) {
|
||||
MemoryPtr Node::prepareWeightMemory(DnnlMemoryDescPtr dstWeightDesc, DnnlMemoryDescPtr srcWeightDesc) {
|
||||
if (!getParentEdgeAt(1)->getParent()->isConstant())
|
||||
IE_THROW() << "Weight input is not const for node " << getName() << ".";
|
||||
auto edgeMem = getParentEdgeAt(1)->getMemoryPtr();
|
||||
if (!edgeMem)
|
||||
IE_THROW() << "Cannot get const weights edgeMem for node " << getName() << ".";
|
||||
|
||||
auto constDnnlMemOutDesc = edgeMem->getDescWithType<DnnlMemoryDesc>();
|
||||
auto weightSrcDesc = constDnnlMemOutDesc->getDnnlDesc();
|
||||
weightSrcDesc = weightSrcDesc.reshape(weightDesc->getDnnlDesc().get_dims());
|
||||
auto create = [&] () {
|
||||
auto newSrcDesc = DnnlExtensionUtils::makeDescriptor(weightSrcDesc);
|
||||
if (!srcWeightDesc) {
|
||||
auto constDnnlMemOutDesc = edgeMem->getDescWithType<DnnlMemoryDesc>();
|
||||
auto weightSrcDesc = constDnnlMemOutDesc->getDnnlDesc();
|
||||
weightSrcDesc = weightSrcDesc.reshape(dstWeightDesc->getDnnlDesc().get_dims());
|
||||
srcWeightDesc = DnnlExtensionUtils::makeDescriptor(weightSrcDesc);
|
||||
}
|
||||
|
||||
Memory srcMemory{ getEngine(), newSrcDesc, edgeMem->getData() };
|
||||
MemoryPtr _ptr = std::make_shared<Memory>(getEngine(), weightDesc);
|
||||
auto create = [&] () {
|
||||
Memory srcMemory{ getEngine(), srcWeightDesc, edgeMem->getData() };
|
||||
MemoryPtr _ptr = std::make_shared<Memory>(getEngine(), dstWeightDesc);
|
||||
node::Reorder::reorderData(srcMemory, *_ptr, context->getParamsCache());
|
||||
|
||||
return _ptr;
|
||||
};
|
||||
|
||||
MemoryPtr ptr;
|
||||
const auto& format = weightDesc->serializeFormat();
|
||||
const auto& format = dstWeightDesc->serializeFormat();
|
||||
auto itr = privateWeightCache.find(format);
|
||||
if (privateWeightCache.end() != itr) {
|
||||
ptr = itr->second;
|
||||
|
||||
@@ -652,7 +652,7 @@ protected:
|
||||
void prepareMemory(const DnnlMemoryDescPtr& intDesc, size_t indx);
|
||||
void prepareMemory(dnnl::primitive_desc_iterator& itpd);
|
||||
|
||||
MemoryPtr prepareWeightMemory(DnnlMemoryDescPtr weightDesc);
|
||||
MemoryPtr prepareWeightMemory(DnnlMemoryDescPtr dstWeightDesc, DnnlMemoryDescPtr srcWeightDesc = nullptr);
|
||||
|
||||
bool isDynamic = false;
|
||||
|
||||
|
||||
@@ -294,12 +294,12 @@ void FullyConnected::prepackMLASWeight() {
|
||||
MemoryPtr ptr;
|
||||
auto create = [&]() {
|
||||
float* weightPtr = reinterpret_cast<float*>(weightsMem->getData());
|
||||
size_t ldb = K;
|
||||
size_t ldb = weightsNonTransposed ? N : K;
|
||||
MemoryPtr _ptr =
|
||||
std::make_shared<Memory>(getEngine(),
|
||||
intel_cpu::CpuBlockedMemoryDesc(Precision::I8, intel_cpu::Shape{packedBsize}));
|
||||
float* prepackedDst = reinterpret_cast<float*>(_ptr->getData());
|
||||
mlas_sgemm_pack("T", N, K, ldb, weightPtr, prepackedDst);
|
||||
mlas_sgemm_pack(weightsNonTransposed ? "F" : "T", N, K, ldb, weightPtr, prepackedDst);
|
||||
return _ptr;
|
||||
};
|
||||
|
||||
@@ -316,7 +316,7 @@ void FullyConnected::prepackMLASWeight() {
|
||||
return ptr;
|
||||
};
|
||||
const auto& wgtDims = getParentEdgeAt(WEIGHTS_ID)->getMemoryPtr()->getStaticDims();
|
||||
// Weight is transpoed by MatMulConstTransposesExtraction
|
||||
// Weights are transposed by MatMulConstTransposesExtraction
|
||||
// K is the IC of weight
|
||||
// the weight is reshaped to [-1, K] in ConvertMatMulToFC
|
||||
K = wgtDims[1];
|
||||
@@ -470,7 +470,11 @@ void FullyConnected::prepareParams() {
|
||||
}
|
||||
|
||||
if (!prevExecPtr || !execPtr->getWeightDesc()->isCompatible(*(prevExecPtr->getWeightDesc()))) {
|
||||
primArgs[DNNL_ARG_WEIGHTS] = prepareWeightMemory(execPtr->getWeightDesc())->getPrimitive();
|
||||
if (weightsNonTransposed) {
|
||||
primArgs[DNNL_ARG_WEIGHTS] = prepareWeightMemory(execPtr->getWeightDesc(), makeTransposedWeightDescriptor())->getPrimitive();
|
||||
} else {
|
||||
primArgs[DNNL_ARG_WEIGHTS] = prepareWeightMemory(execPtr->getWeightDesc())->getPrimitive();
|
||||
}
|
||||
}
|
||||
// changed shapes may also cause the kernel type changed
|
||||
selected_pd->setImplementationType(execPtr->getImplementationType());
|
||||
@@ -1074,6 +1078,22 @@ void FullyConnected::fuseDecompressionConstant(const NodePtr& constData, std::ve
|
||||
Precision::FP32,
|
||||
elementsCount);
|
||||
}
|
||||
|
||||
DnnlMemoryDescPtr FullyConnected::makeTransposedWeightDescriptor() {
|
||||
if (!getParentEdgeAt(1)->getParent()->isConstant())
|
||||
IE_THROW() << "Weight input is not const for node " << getName() << ".";
|
||||
auto edgeMem = getParentEdgeAt(1)->getMemoryPtr();
|
||||
if (!edgeMem)
|
||||
IE_THROW() << "Cannot get const weights edgeMem for node " << getName() << ".";
|
||||
|
||||
auto constDnnlMemOutDesc = edgeMem->getDescWithType<DnnlMemoryDesc>();
|
||||
auto weightSrcDesc = constDnnlMemOutDesc->getDnnlDesc();
|
||||
weightSrcDesc = {weightSrcDesc.get_dims(), weightSrcDesc.get_data_type(), memory::format_tag::ba};
|
||||
weightSrcDesc = weightSrcDesc.reshape(execPtr->getWeightDesc()->getDnnlDesc().get_dims());
|
||||
|
||||
return DnnlExtensionUtils::makeDescriptor(weightSrcDesc);
|
||||
}
|
||||
|
||||
} // namespace node
|
||||
} // namespace intel_cpu
|
||||
} // namespace ov
|
||||
|
||||
@@ -56,6 +56,9 @@ public:
|
||||
void prepareParams() override;
|
||||
void executeDynamicImpl(dnnl::stream strm) override;
|
||||
bool canBeExecutedInInt8() const override;
|
||||
void keepWeightsNonTransposed(bool weightsNonTransposed) {
|
||||
this->weightsNonTransposed = weightsNonTransposed;
|
||||
}
|
||||
|
||||
void fuseDecompressionMultiply(const NodePtr& constData);
|
||||
const std::vector<float>& getDecompressionMultiply() const { return decompressionMultiply; }
|
||||
@@ -118,6 +121,10 @@ private:
|
||||
bool useWeightsDecompressionImpl = false;
|
||||
std::vector<float> decompressionSubtract;
|
||||
std::vector<float> decompressionMultiply;
|
||||
|
||||
// FC with transposed weights
|
||||
bool weightsNonTransposed = false;
|
||||
DnnlMemoryDescPtr makeTransposedWeightDescriptor();
|
||||
};
|
||||
|
||||
} // namespace node
|
||||
|
||||
@@ -76,7 +76,7 @@ void Transpose::initSupportedPrimitiveDescriptors() {
|
||||
config.inConfs[INPUT_ORDER_IDX].constant(isInputOrderConst);
|
||||
config.inConfs[INPUT_ORDER_IDX].setMemDesc(creatorsMap.at(LayoutType::ncsp)->createSharedDesc(
|
||||
Precision::I32, getInputShapeAtPort(INPUT_ORDER_IDX)));
|
||||
config.outConfs[0].inPlace(-1);
|
||||
config.outConfs[0].inPlace(isOptimized ? 0 : -1);
|
||||
config.outConfs[0].constant(false);
|
||||
transpose_context = std::make_shared<ExecutorContext>(context, getImplPriority());
|
||||
|
||||
@@ -125,7 +125,7 @@ void Transpose::initSupportedPrimitiveDescriptors() {
|
||||
}
|
||||
|
||||
bool Transpose::isExecutable() const {
|
||||
return !isInputTensorAtPortEmpty(0);
|
||||
return !isInputTensorAtPortEmpty(0) && !isOptimized;
|
||||
}
|
||||
|
||||
bool Transpose::needPrepareParams() const {
|
||||
@@ -133,6 +133,9 @@ bool Transpose::needPrepareParams() const {
|
||||
}
|
||||
|
||||
void Transpose::prepareParams() {
|
||||
if (isOptimized)
|
||||
return;
|
||||
|
||||
if (performAsReorder) {
|
||||
// Transpose(order={0,3,1,2}) can be performed as Reorder(acdb=>abcd)
|
||||
auto srcMemPtr = getParentEdgeAt(INPUT_DATA_IDX)->getMemoryPtr();
|
||||
@@ -191,6 +194,9 @@ void Transpose::prepareParams() {
|
||||
}
|
||||
|
||||
void Transpose::createPrimitive() {
|
||||
if (isOptimized)
|
||||
return;
|
||||
|
||||
auto dstMemPtr = getChildEdgeAt(0)->getMemoryPtr();
|
||||
auto srcMemPtr = getParentEdgeAt(INPUT_DATA_IDX)->getMemoryPtr();
|
||||
if (!dstMemPtr || !dstMemPtr->isAllocated())
|
||||
@@ -223,6 +229,9 @@ void Transpose::createPrimitive() {
|
||||
}
|
||||
|
||||
void Transpose::execute(dnnl::stream strm) {
|
||||
if (isOptimized)
|
||||
return;
|
||||
|
||||
if (prim) {
|
||||
prim.execute(strm, primArgs);
|
||||
} else if (execPtr) {
|
||||
|
||||
@@ -38,6 +38,10 @@ public:
|
||||
bool needPrepareParams() const override;
|
||||
void prepareParams() override;
|
||||
|
||||
void setOptimized(bool isOptimized) {
|
||||
this->isOptimized = isOptimized;
|
||||
}
|
||||
|
||||
protected:
|
||||
void executeDynamicImpl(dnnl::stream strm) override;
|
||||
std::shared_ptr<ExecutorContext> transpose_context;
|
||||
@@ -56,6 +60,7 @@ private:
|
||||
static constexpr size_t INPUT_ORDER_IDX = 1lu;
|
||||
|
||||
bool performAsReorder = false;
|
||||
bool isOptimized = false;
|
||||
};
|
||||
|
||||
} // namespace node
|
||||
|
||||
+2
-1
@@ -113,12 +113,13 @@ ov::intel_cpu::ConvertMatMulToFC::ConvertMatMulToFC() {
|
||||
std::swap(*(transpose_order.end() - 1), *(transpose_order.end() - 2));
|
||||
|
||||
auto transpose_const = ngraph::op::v0::Constant::create(ngraph::element::i32, ngraph::Shape{ transpose_order.size() }, transpose_order);
|
||||
auto transpose = ov::op::util::make_try_fold<ngraph::op::v1::Transpose>(node, transpose_const);
|
||||
auto transpose = std::make_shared<ngraph::op::v1::Transpose>(node, transpose_const);
|
||||
if (!ngraph::is_type<ngraph::op::v0::Constant>(transpose)) {
|
||||
new_ops.push_back(transpose_const);
|
||||
MatcherPass::register_new_node(transpose);
|
||||
}
|
||||
transpose->set_friendly_name(transpose_name);
|
||||
ov::disable_constant_folding(transpose);
|
||||
new_ops.push_back(transpose);
|
||||
return transpose;
|
||||
};
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
#include "transformations/common_optimizations/augru_cell_fusion.hpp"
|
||||
#include "transformations/common_optimizations/common_optimizations.hpp"
|
||||
#include "transformations/common_optimizations/wrap_interpolate_into_transposes.hpp"
|
||||
#include "transformations/common_optimizations/matmul_const_transposes_extraction.hpp"
|
||||
#include "transformations/control_flow/unroll_tensor_iterator.hpp"
|
||||
#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp"
|
||||
#include "transformations/op_conversions/convert_batch_to_space.hpp"
|
||||
@@ -399,6 +400,7 @@ void Transformations::PreLpt(const std::vector<ov::element::Type>& defaultPrecis
|
||||
CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertTopK3);
|
||||
CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertTopK11ToTopK3);
|
||||
CPU_DISABLE_PASS_COMMON(manager, ov::pass::HSwishDecomposition);
|
||||
CPU_DISABLE_PASS_COMMON(manager, ov::pass::MatMulConstTransposesExtraction);
|
||||
CPU_DISABLE_PASS_X64(manager, ov::pass::HSigmoidDecomposition);
|
||||
|
||||
CPU_DISABLE_PASS_X64(manager, ov::pass::ReduceL1Decomposition);
|
||||
|
||||
+214
-36
@@ -14,9 +14,11 @@ using namespace ov::test;
|
||||
|
||||
namespace SubgraphTestsDefinitions {
|
||||
|
||||
/* This test checks that the ConvertMatMulToFC transformation should work and the MatMul node is converted to the FC node.
|
||||
* The Convert node should be removed on the CPU plugin side.
|
||||
/* This test checks MatMul weights constant folding on CPU plugin side and cover two optimizations:
|
||||
1. Decompressing Convert FP16 -> FP32 CF (FuseFCAndConvertOnWeights in cpu graph optimizer)
|
||||
2. Transpose CF (FuseFCAndTransposeOnWeights in cpu graph optimizer)
|
||||
|
||||
* 1. Graph with decompressing Convert FP16 -> FP32. The Convert node should be removed on the CPU plugin side.
|
||||
* Graph before:
|
||||
------------ ------------
|
||||
|Input(f32)| |Input(f16)|
|
||||
@@ -46,12 +48,46 @@ namespace SubgraphTestsDefinitions {
|
||||
--------
|
||||
|Output|
|
||||
--------
|
||||
|
||||
* 2. Graph with Transpose. In case of (transpose_b == false), ConvertMatMulToFC() transformation should insert Transpose on weights.
|
||||
* It must not fold and must remain in the execution graph.
|
||||
* Graph before:
|
||||
------------ ------------
|
||||
|Input(f32)| |Input(f32)|
|
||||
------------ ------------
|
||||
| |
|
||||
-------------------------------------
|
||||
| MatMul(transpose_b == false) |
|
||||
-------------------------------------
|
||||
|
|
||||
--------
|
||||
|Output|
|
||||
--------
|
||||
|
||||
* Exec graph:
|
||||
------------ ------------
|
||||
|Input(f32)| |Input(f32)|
|
||||
------------ ------------
|
||||
| |
|
||||
| -------------
|
||||
| | Transpose |
|
||||
| -------------
|
||||
| |
|
||||
----------------------------
|
||||
| FullyConnected |
|
||||
----------------------------
|
||||
|
|
||||
--------
|
||||
|Output|
|
||||
--------
|
||||
*/
|
||||
|
||||
using MatMulDecompressConvertParams = std::tuple<
|
||||
std::vector<InputShape>, // input shapes
|
||||
std::pair<bool, bool>, // transposeA, transposeB
|
||||
std::map<std::string, std::string> // additional config
|
||||
std::vector<InputShape>, // input shapes
|
||||
std::pair<bool, bool>, // transposeA, transposeB
|
||||
ElementType, // weights precision
|
||||
std::map<std::string, std::string>, // additional config
|
||||
CPUSpecificParams
|
||||
>;
|
||||
|
||||
class MatMulDecompressConvertTest : public testing::WithParamInterface<MatMulDecompressConvertParams>,
|
||||
@@ -60,9 +96,11 @@ public:
|
||||
static std::string getTestCaseName(testing::TestParamInfo<MatMulDecompressConvertParams> obj) {
|
||||
std::vector<InputShape> inputShapes;
|
||||
std::pair<bool, bool> transpose;
|
||||
ElementType weiElemType;
|
||||
std::map<std::string, std::string> additionalConfig;
|
||||
CPUSpecificParams cpuParams;
|
||||
|
||||
std::tie(inputShapes, transpose, additionalConfig) = obj.param;
|
||||
std::tie(inputShapes, transpose, weiElemType, additionalConfig, cpuParams) = obj.param;
|
||||
|
||||
std::ostringstream result;
|
||||
for (const auto& shape : inputShapes) {
|
||||
@@ -82,12 +120,16 @@ public:
|
||||
result << "transpose_a=" << transpose.first << "_";
|
||||
result << "transpose_b=" << transpose.second << "_";
|
||||
|
||||
result << "weiLemType=" << weiElemType << "_";
|
||||
|
||||
result << "config=(";
|
||||
for (const auto& configEntry : additionalConfig) {
|
||||
result << configEntry.first << ", " << configEntry.second << ":";
|
||||
}
|
||||
result << ")";
|
||||
|
||||
result << CPUTestsBase::getTestCaseName(cpuParams);
|
||||
|
||||
return result.str();
|
||||
}
|
||||
|
||||
@@ -98,7 +140,7 @@ protected:
|
||||
std::swap(*(shape.end() - 1), *(shape.end() - 2));
|
||||
}
|
||||
|
||||
void CheckConstFP16() const {
|
||||
void CheckFCWeightsPrecision() const {
|
||||
auto getExecValue = [](const ov::Node::RTMap& rtInfo, const std::string ¶mName) -> std::string {
|
||||
auto it = rtInfo.find(paramName);
|
||||
IE_ASSERT(rtInfo.end() != it);
|
||||
@@ -110,8 +152,8 @@ protected:
|
||||
for (const auto &fcNode : execFunction->get_ops()) {
|
||||
if (getExecValue(fcNode->get_rt_info(), ExecGraphInfoSerialization::LAYER_TYPE) == "FullyConnected") {
|
||||
const auto &constNode = fcNode->get_input_node_shared_ptr(1);
|
||||
ASSERT_EQ(getExecValue(constNode->get_rt_info(), ExecGraphInfoSerialization::LAYER_TYPE), "Const");
|
||||
ASSERT_EQ(getExecValue(constNode->get_rt_info(), ExecGraphInfoSerialization::OUTPUT_PRECISIONS), "FP16");
|
||||
element::Type expectedType(getExecValue(constNode->get_rt_info(), ExecGraphInfoSerialization::OUTPUT_PRECISIONS));
|
||||
ASSERT_EQ(expectedType, weiConstElemType);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -122,14 +164,19 @@ protected:
|
||||
std::vector<InputShape> inputShapes;
|
||||
std::pair<bool, bool> transpose;
|
||||
std::map<std::string, std::string> additionalConfig;
|
||||
CPUSpecificParams cpuParams;
|
||||
|
||||
std::tie(inputShapes, transpose, additionalConfig) = this->GetParam();
|
||||
std::tie(inputShapes, transpose, weiConstElemType, additionalConfig, cpuParams) = this->GetParam();
|
||||
std::tie(inFmts, outFmts, priority, selectedType) = cpuParams;
|
||||
|
||||
init_input_shapes(inputShapes);
|
||||
|
||||
bool transpA = transpose.first;
|
||||
bool transpB = transpose.second;
|
||||
|
||||
if (transpA) transposesCount++;
|
||||
if (!transpB) transposesCount++;
|
||||
|
||||
if (transpA) {
|
||||
transposeShape(inputDynamicShapes[0]);
|
||||
for (auto& shapes : targetStaticShapes) {
|
||||
@@ -148,32 +195,47 @@ protected:
|
||||
|
||||
configuration.insert(additionalConfig.begin(), additionalConfig.end());
|
||||
|
||||
ElementType netType = element::f32;
|
||||
if (additionalConfig[PluginConfigParams::KEY_ENFORCE_BF16] == PluginConfigParams::YES)
|
||||
inType = outType = netType = ElementType::bf16;
|
||||
else
|
||||
ElementType netType = ElementType::f32;
|
||||
ElementType convertOutType = ElementType::f32;
|
||||
if (additionalConfig[PluginConfigParams::KEY_ENFORCE_BF16] == PluginConfigParams::YES) {
|
||||
convertOutType = inType = outType = netType = ElementType::bf16;
|
||||
weiConstElemType = (weiConstElemType != ElementType::f32) ? weiConstElemType : ElementType::bf16;
|
||||
} else {
|
||||
inType = outType = netType;
|
||||
}
|
||||
|
||||
std::string cpuNodeType = "FullyConnected";
|
||||
selectedType = makeSelectedTypeStr(selectedType, outType);
|
||||
|
||||
auto params = builder::makeDynamicParams(inType, {inShapeA});
|
||||
auto paramOuts = helpers::convert2OutputVector(helpers::castOps2Nodes<opset1::Parameter>(params));
|
||||
|
||||
auto matrixB = ngraph::builder::makeConstant<float16>(element::f16, inShapeB.get_shape(), {}, true);
|
||||
auto convert = std::make_shared<ngraph::opset1::Convert>(matrixB, inType);
|
||||
mark_as_decompression(convert);
|
||||
auto matMul = builder::makeMatMul(paramOuts[0], convert, transpA, transpB);
|
||||
std::shared_ptr<Node> inputB = builder::makeConstant<float>(weiConstElemType, inShapeB.get_shape(), {}, true);
|
||||
if (weiConstElemType == ElementType::f16) {
|
||||
inputB = std::make_shared<opset1::Convert>(inputB, convertOutType);
|
||||
mark_as_decompression(inputB);
|
||||
}
|
||||
auto matMul = builder::makeMatMul(paramOuts[0], inputB, transpA, transpB);
|
||||
|
||||
function = CPUTestsBase::makeNgraphFunction(netType, params, matMul, cpuNodeType);
|
||||
}
|
||||
|
||||
void CheckExecutionGraph() {
|
||||
CheckPluginRelatedResults(compiledModel, "FullyConnected");
|
||||
CheckNumberOfNodesWithType(compiledModel, "FullyConnected", 1);
|
||||
CheckNumberOfNodesWithType(compiledModel, "Transpose", transposesCount);
|
||||
CheckNumberOfNodesWithType(compiledModel, "Convert", 0);
|
||||
CheckNumberOfNodesWithType(compiledModel, "Reorder", 0);
|
||||
CheckFCWeightsPrecision();
|
||||
}
|
||||
|
||||
size_t transposesCount = 0;
|
||||
ElementType weiConstElemType = ElementType::f32;
|
||||
};
|
||||
|
||||
TEST_P(MatMulDecompressConvertTest, CompareWithRefs) {
|
||||
SKIP_IF_CURRENT_TEST_IS_DISABLED();
|
||||
run();
|
||||
CheckNumberOfNodesWithType(compiledModel, "FullyConnected", 1);
|
||||
CheckNumberOfNodesWithType(compiledModel, "Convert", 0);
|
||||
CheckNumberOfNodesWithType(compiledModel, "Reorder", 0);
|
||||
CheckConstFP16();
|
||||
CheckExecutionGraph();
|
||||
}
|
||||
|
||||
namespace {
|
||||
@@ -185,32 +247,148 @@ const std::vector<std::pair<bool, bool>> transposeParams = {
|
||||
{true, true},
|
||||
};
|
||||
|
||||
const std::vector<std::vector<InputShape>> inputShapes2D = {
|
||||
static_shapes_to_test_representation({{2, 3}, {3, 4}}),
|
||||
{
|
||||
{{-1, -1}, {{2, 3}, {5, 3}}},
|
||||
{{3, 4}, {{3, 4}, {3, 4}}}
|
||||
},
|
||||
};
|
||||
|
||||
const std::vector<std::vector<InputShape>> inputShapes3D = {
|
||||
static_shapes_to_test_representation({{2, 2, 3}, {3, 4}}),
|
||||
static_shapes_to_test_representation({{2, 3}, {1, 3, 4}}),
|
||||
static_shapes_to_test_representation({{1, 2, 3}, {1, 3, 4}}),
|
||||
{
|
||||
{{-1, -1, -1}, {{2, 2, 3}, {3, 5, 3}}},
|
||||
{{3, 4}, {{3, 4}, {3, 4}}}
|
||||
},
|
||||
{
|
||||
{{-1, -1}, {{2, 3}, {5, 3}}},
|
||||
{{1, 3, 4}, {{1, 3, 4}, {1, 3, 4}}}
|
||||
},
|
||||
{
|
||||
{{-1, -1, -1}, {{1, 2, 3}, {1, 5, 3}}},
|
||||
{{1, 3, 4}, {{1, 3, 4}, {1, 3, 4}}}
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
std::vector<std::map<std::string, std::string>> filterAdditionalConfig() {
|
||||
std::vector<std::map<std::string, std::string>> additionalConfig;
|
||||
#ifndef OV_CPU_WITH_MLAS
|
||||
additionalConfig.push_back(std::map<std::string, std::string>{/* empty config */});
|
||||
if (with_cpu_x86_avx512_core()) {
|
||||
additionalConfig.push_back({{PluginConfigParams::KEY_ENFORCE_BF16, PluginConfigParams::YES}});
|
||||
}
|
||||
|
||||
#endif
|
||||
return additionalConfig;
|
||||
}
|
||||
|
||||
std::vector<std::map<std::string, std::string>> filterAdditionalConfig_BF16() {
|
||||
std::vector<std::map<std::string, std::string>> additionalConfig;
|
||||
if (with_cpu_x86_avx512_core()) {
|
||||
additionalConfig.push_back({{PluginConfigParams::KEY_ENFORCE_BF16, PluginConfigParams::YES}});
|
||||
}
|
||||
return additionalConfig;
|
||||
}
|
||||
|
||||
std::vector<std::map<std::string, std::string>> filterAdditionalConfig_MLAS() {
|
||||
std::vector<std::map<std::string, std::string>> additionalConfig;
|
||||
additionalConfig.push_back(std::map<std::string, std::string>{/* empty config */});
|
||||
return additionalConfig;
|
||||
}
|
||||
|
||||
std::vector<CPUSpecificParams> filterSpecificParams() {
|
||||
std::vector<CPUSpecificParams> specificParams;
|
||||
if (with_cpu_x86_avx512_core()) {
|
||||
specificParams.push_back(CPUSpecificParams{{}, {}, {"brgemm_avx512"}, "brgemm_avx512"});
|
||||
} else if (with_cpu_x86_avx2()) {
|
||||
specificParams.push_back(CPUSpecificParams{{}, {}, {"brgemm_avx2"}, "brgemm_avx2"});
|
||||
}
|
||||
return specificParams;
|
||||
}
|
||||
|
||||
|
||||
std::vector<CPUSpecificParams> filterSpecificParams_BF16() {
|
||||
std::vector<CPUSpecificParams> specificParams;
|
||||
specificParams.push_back(CPUSpecificParams{{}, {}, {"jit_gemm"}, "jit_gemm"});
|
||||
return specificParams;
|
||||
}
|
||||
|
||||
|
||||
std::vector<CPUSpecificParams> filterSpecificParams_MLAS() {
|
||||
std::vector<CPUSpecificParams> specificParams;
|
||||
specificParams.push_back(CPUSpecificParams{{}, {}, {"gemm_mlas"}, "gemm_mlas"});
|
||||
return specificParams;
|
||||
}
|
||||
|
||||
|
||||
#ifdef OV_CPU_WITH_MLAS
|
||||
const auto testParams2D_MLAS_smoke = ::testing::Combine(
|
||||
::testing::ValuesIn(inputShapes2D),
|
||||
::testing::ValuesIn(transposeParams),
|
||||
::testing::Values(ElementType::f32),
|
||||
::testing::ValuesIn(filterAdditionalConfig_MLAS()),
|
||||
::testing::ValuesIn(filterSpecificParams_MLAS()));
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_MLAS, MatMulDecompressConvertTest, testParams2D_MLAS_smoke,
|
||||
MatMulDecompressConvertTest::getTestCaseName);
|
||||
#endif
|
||||
|
||||
|
||||
const auto testParams2D_smoke = ::testing::Combine(
|
||||
::testing::Values(static_shapes_to_test_representation({{2, 3}, {3, 4}})),
|
||||
::testing::ValuesIn(transposeParams),
|
||||
::testing::ValuesIn(filterAdditionalConfig()));
|
||||
::testing::ValuesIn(inputShapes2D),
|
||||
::testing::ValuesIn(transposeParams),
|
||||
::testing::Values(ElementType::f32, ElementType::f16),
|
||||
::testing::ValuesIn(filterAdditionalConfig()),
|
||||
::testing::ValuesIn(filterSpecificParams()));
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FC_2D, MatMulDecompressConvertTest, testParams2D_smoke,
|
||||
MatMulDecompressConvertTest::getTestCaseName);
|
||||
MatMulDecompressConvertTest::getTestCaseName);
|
||||
|
||||
|
||||
const auto testParams2D_BF16_smoke = ::testing::Combine(
|
||||
::testing::ValuesIn(inputShapes2D),
|
||||
::testing::ValuesIn(transposeParams),
|
||||
::testing::Values(ElementType::f32, ElementType::f16),
|
||||
::testing::ValuesIn(filterAdditionalConfig_BF16()),
|
||||
::testing::ValuesIn(filterSpecificParams_BF16()));
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_BF16, MatMulDecompressConvertTest, testParams2D_BF16_smoke,
|
||||
MatMulDecompressConvertTest::getTestCaseName);
|
||||
|
||||
|
||||
#ifdef OV_CPU_WITH_MLAS
|
||||
const auto testParams3D_MLAS_smoke = ::testing::Combine(
|
||||
::testing::ValuesIn(inputShapes3D),
|
||||
::testing::ValuesIn(transposeParams),
|
||||
::testing::Values(ElementType::f32),
|
||||
::testing::ValuesIn(filterAdditionalConfig_MLAS()),
|
||||
::testing::ValuesIn(filterSpecificParams_MLAS()));
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FC_3D_MLAS, MatMulDecompressConvertTest, testParams3D_MLAS_smoke,
|
||||
MatMulDecompressConvertTest::getTestCaseName);
|
||||
#endif
|
||||
|
||||
|
||||
const auto testParams3D_smoke = ::testing::Combine(
|
||||
::testing::Values(static_shapes_to_test_representation({{1, 2, 3}, {3, 4}}),
|
||||
static_shapes_to_test_representation({{2, 3}, {1, 3, 4}})),
|
||||
::testing::ValuesIn(transposeParams),
|
||||
::testing::ValuesIn(filterAdditionalConfig()));
|
||||
::testing::ValuesIn(inputShapes3D),
|
||||
::testing::ValuesIn(transposeParams),
|
||||
::testing::Values(ElementType::f32, ElementType::f16),
|
||||
::testing::ValuesIn(filterAdditionalConfig()),
|
||||
::testing::ValuesIn(filterSpecificParams()));
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FC_3D, MatMulDecompressConvertTest, testParams3D_smoke,
|
||||
MatMulDecompressConvertTest::getTestCaseName);
|
||||
MatMulDecompressConvertTest::getTestCaseName);
|
||||
|
||||
|
||||
const auto testParams3D_BF16_smoke = ::testing::Combine(
|
||||
::testing::ValuesIn(inputShapes3D),
|
||||
::testing::ValuesIn(transposeParams),
|
||||
::testing::Values(ElementType::f32, ElementType::f16),
|
||||
::testing::ValuesIn(filterAdditionalConfig_BF16()),
|
||||
::testing::ValuesIn(filterSpecificParams_BF16()));
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FC_3D_BF16, MatMulDecompressConvertTest, testParams3D_BF16_smoke,
|
||||
MatMulDecompressConvertTest::getTestCaseName);
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
@@ -35,10 +35,14 @@ TEST_F(TransformationTestsF, ConvertMatMulToFCTest1) {
|
||||
}
|
||||
{
|
||||
auto input1 = std::make_shared<ngraph::opset1::Parameter>(ngraph::element::f32, ngraph::Shape{ 3, 2, 2 });
|
||||
auto transpose_constant = ngraph::opset1::Constant::create(ngraph::element::i32, ngraph::Shape{ 3 }, { 0, 2, 1 });
|
||||
auto transpose = std::make_shared<ngraph::opset1::Transpose>(input1, transpose_constant);
|
||||
auto transpose_constant1 = ngraph::opset1::Constant::create(ngraph::element::i32, ngraph::Shape{ 3 }, { 0, 2, 1 });
|
||||
auto transpose1 = std::make_shared<ngraph::opset1::Transpose>(input1, transpose_constant1);
|
||||
|
||||
auto input2 = ngraph::opset1::Constant::create(ngraph::element::f32, ngraph::Shape{ 2, 2 }, { 1 });
|
||||
auto matmul = std::make_shared<FullyConnectedNode>(transpose, input2, ngraph::Rank(3));
|
||||
auto transpose_constant2 = ngraph::opset1::Constant::create(ngraph::element::i32, ngraph::Shape{ 2 }, { 1, 0 });
|
||||
auto transpose2 = std::make_shared<ngraph::opset1::Transpose>(input2, transpose_constant2);
|
||||
|
||||
auto matmul = std::make_shared<FullyConnectedNode>(transpose1, transpose2, ngraph::Rank(3));
|
||||
|
||||
function_ref = std::make_shared<ngraph::Function>(ngraph::NodeVector{ matmul }, ngraph::ParameterVector{ input1 });
|
||||
}
|
||||
@@ -311,8 +315,12 @@ TEST_F(TransformationTestsF, ConvertMatMulToFCTest_decompress_convert_0) {
|
||||
}
|
||||
{
|
||||
auto input1 = std::make_shared<ngraph::opset1::Parameter>(ngraph::element::f32, ngraph::Shape{ 3, 2, 2 });
|
||||
auto input2 = ngraph::opset1::Constant::create(ngraph::element::f16, ngraph::Shape{2, 2 }, { 1 });
|
||||
auto convert = std::make_shared<ngraph::opset1::Convert>(input2, ngraph::element::f32);
|
||||
|
||||
auto input2 = ngraph::opset1::Constant::create(ngraph::element::f16, ngraph::Shape{ 2, 2 }, { 1 });
|
||||
auto transpose_constant = ngraph::opset1::Constant::create(ngraph::element::i32, ngraph::Shape{ 2 }, { 1, 0 });
|
||||
auto transpose = std::make_shared<ngraph::opset1::Transpose>(input2, transpose_constant);
|
||||
auto convert = std::make_shared<ngraph::opset1::Convert>(transpose, ngraph::element::f32);
|
||||
|
||||
auto matmul = std::make_shared<FullyConnectedNode>(input1, convert, ngraph::Rank(3));
|
||||
|
||||
function_ref = std::make_shared<ngraph::Function>(ngraph::NodeVector{ matmul }, ngraph::ParameterVector{ input1 });
|
||||
@@ -332,11 +340,15 @@ TEST_F(TransformationTestsF, ConvertMatMulToFCTest_decompress_convert_1) {
|
||||
}
|
||||
{
|
||||
auto input1 = std::make_shared<ngraph::opset1::Parameter>(ngraph::element::f32, ngraph::Shape{ 3, 2, 2 });
|
||||
auto transpose_constant = ngraph::opset1::Constant::create(ngraph::element::i32, ngraph::Shape{ 3 }, { 0, 2, 1 });
|
||||
auto transpose = std::make_shared<ngraph::opset1::Transpose>(input1, transpose_constant);
|
||||
auto input2 = ngraph::opset1::Constant::create(ngraph::element::f16, ngraph::Shape{2, 2 }, { 1 });
|
||||
auto convert = std::make_shared<ngraph::opset1::Convert>(input2, ngraph::element::f32);
|
||||
auto matmul = std::make_shared<FullyConnectedNode>(transpose, convert, ngraph::Rank(3));
|
||||
auto transpose_constant1 = ngraph::opset1::Constant::create(ngraph::element::i32, ngraph::Shape{ 3 }, { 0, 2, 1 });
|
||||
auto transpose1 = std::make_shared<ngraph::opset1::Transpose>(input1, transpose_constant1);
|
||||
|
||||
auto input2 = ngraph::opset1::Constant::create(ngraph::element::f16, ngraph::Shape{ 2, 2 }, { 1 });
|
||||
auto transpose_constant2 = ngraph::opset1::Constant::create(ngraph::element::i32, ngraph::Shape{ 2 }, { 1, 0 });
|
||||
auto transpose2 = std::make_shared<ngraph::opset1::Transpose>(input2, transpose_constant2);
|
||||
auto convert = std::make_shared<ngraph::opset1::Convert>(transpose2, ngraph::element::f32);
|
||||
|
||||
auto matmul = std::make_shared<FullyConnectedNode>(transpose1, convert, ngraph::Rank(3));
|
||||
|
||||
function_ref = std::make_shared<ngraph::Function>(ngraph::NodeVector{ matmul }, ngraph::ParameterVector{ input1 });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user