diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp index e5f3b84f71e..a8fbf42adeb 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp @@ -35,10 +35,13 @@ using namespace InferenceEngine; MKLDNNBinaryConvolutionNode::MKLDNNBinaryConvolutionNode(const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache) - : MKLDNNNode(layer, eng, cache) { + : MKLDNNNode(layer, eng, cache), baseInputsNumber(1) { internalBlobDesc.emplace_back([&](primitive_desc_iterator &primitive_desc_it, size_t idx) -> MKLDNNMemoryDesc { return MKLDNNMemoryDesc(primitive_desc_it.weights_primitive_desc(0).desc()); }); + + if (getCnnLayer()->type == "BinaryConvolution") + baseInputsNumber = getCnnLayer().get()->insData.size(); } void MKLDNNBinaryConvolutionNode::getSupportedDescriptors() { @@ -135,7 +138,7 @@ void MKLDNNBinaryConvolutionNode::getSupportedDescriptors() { #endif } - int expectedInputEdgesNum = 1 + isFusedWith(Eltwise); + int expectedInputEdgesNum = baseInputsNumber + isFusedWith(Eltwise); for (int i = 0; i < fusedWith.size(); i++) { auto *convolutionNode = dynamic_cast(fusedWith[i].get()); if (convolutionNode) { @@ -319,8 +322,8 @@ void MKLDNNBinaryConvolutionNode::setPostOps(mkldnn::primitive_attr &attr, bool ops.append_dw_conv(dw_conv_ih, dw_conv_iw, dw_conv_kernel[Y_AXIS], dw_conv_kernel[X_AXIS], dw_conv_strides[Y_AXIS], dw_conv_strides[X_AXIS], mkldnn::memory::convert_to_c(dw_conv_in_dt), - static_cast(getParentEdgeAt(1)->getMemory().GetData()), - static_cast(getParentEdgeAt(2)->getMemory().GetData())); + static_cast(getParentEdgeAt(baseInputsNumber + 0)->getMemory().GetData()), + static_cast(getParentEdgeAt(baseInputsNumber + 1)->getMemory().GetData())); } } else { ops.append_dw_conv(dw_conv_ih, dw_conv_iw, dw_conv_kernel[Y_AXIS], dw_conv_kernel[X_AXIS], @@ -363,22 +366,32 @@ void MKLDNNBinaryConvolutionNode::initSupportedPrimitiveDescriptors() { } if (withDWConv) { - auto weightsPrc = memory::data_type::f32; - auto biasPrc = memory::data_type::f32; + int convNumInput = 1; + for (auto &node : fusedWith) { + auto* convolutionNode = dynamic_cast(node.get()); + if (convolutionNode) { + convNumInput = convolutionNode->getBaseIntputsNumber(); + break; + } + } + if (convNumInput > 1) { + auto weightsPrc = memory::data_type::f32; + auto biasPrc = memory::data_type::f32; - MKLDNNDims dwWeightsDims({dw_conv_oc, (ptrdiff_t)1, (ptrdiff_t)1, dw_conv_kernel[Y_AXIS], dw_conv_kernel[X_AXIS]}); - MKLDNNDims dwBiasesDims({dw_conv_oc}); - auto w_fmt = mkldnn::impl::cpu::mayiuse(impl::cpu::cpu_isa_t::avx512_common) - ? memory::format::Goihw16g : memory::format::Goihw8g; + MKLDNNDims dwWeightsDims({dw_conv_oc, (ptrdiff_t)1, (ptrdiff_t)1, dw_conv_kernel[Y_AXIS], dw_conv_kernel[X_AXIS]}); + MKLDNNDims dwBiasesDims({dw_conv_oc}); + auto w_fmt = mkldnn::impl::cpu::mayiuse(impl::cpu::cpu_isa_t::avx512_common) + ? memory::format::Goihw16g : memory::format::Goihw8g; - InferenceEngine::DataConfig dataConfig; - dataConfig.inPlace = -1; - dataConfig.constant = false; - dataConfig.desc = MKLDNNMemoryDesc(dwWeightsDims, weightsPrc, w_fmt); - config.inConfs.push_back(dataConfig); + InferenceEngine::DataConfig dataConfig; + dataConfig.inPlace = -1; + dataConfig.constant = false; + dataConfig.desc = MKLDNNMemoryDesc(dwWeightsDims, weightsPrc, w_fmt); + config.inConfs.push_back(dataConfig); - dataConfig.desc = MKLDNNMemoryDesc(dwBiasesDims, biasPrc, memory::format::x); - config.inConfs.push_back(dataConfig); + dataConfig.desc = MKLDNNMemoryDesc(dwBiasesDims, biasPrc, memory::format::x); + config.inConfs.push_back(dataConfig); + } } std::vector outFormats; @@ -481,22 +494,32 @@ void MKLDNNBinaryConvolutionNode::initDescriptor(const InferenceEngine::LayerCon } if (withDWConv) { - auto weightsPrc = memory::data_type::f32; - auto biasPrc = memory::data_type::f32; + int convNumInput = 1; + for (auto &node : fusedWith) { + auto* convolutionNode = dynamic_cast(node.get()); + if (convolutionNode) { + convNumInput = convolutionNode->getBaseIntputsNumber(); + break; + } + } + if (convNumInput > 1) { + auto weightsPrc = memory::data_type::f32; + auto biasPrc = memory::data_type::f32; - MKLDNNDims dwWeightsDims({dw_conv_oc, (ptrdiff_t)1, (ptrdiff_t)1, dw_conv_kernel[Y_AXIS], dw_conv_kernel[X_AXIS]}); - MKLDNNDims dwBiasesDims({dw_conv_oc}); - auto w_fmt = mkldnn::impl::cpu::mayiuse(impl::cpu::cpu_isa_t::avx512_common) - ? memory::format::Goihw16g : memory::format::Goihw8g; + MKLDNNDims dwWeightsDims({dw_conv_oc, (ptrdiff_t)1, (ptrdiff_t)1, dw_conv_kernel[Y_AXIS], dw_conv_kernel[X_AXIS]}); + MKLDNNDims dwBiasesDims({dw_conv_oc}); + auto w_fmt = mkldnn::impl::cpu::mayiuse(impl::cpu::cpu_isa_t::avx512_common) + ? memory::format::Goihw16g : memory::format::Goihw8g; - InferenceEngine::DataConfig dataConfig; - dataConfig.inPlace = -1; - dataConfig.constant = false; - dataConfig.desc = MKLDNNMemoryDesc(dwWeightsDims, weightsPrc, w_fmt); - cfg.inConfs.push_back(dataConfig); + InferenceEngine::DataConfig dataConfig; + dataConfig.inPlace = -1; + dataConfig.constant = false; + dataConfig.desc = MKLDNNMemoryDesc(dwWeightsDims, weightsPrc, w_fmt); + cfg.inConfs.push_back(dataConfig); - dataConfig.desc = MKLDNNMemoryDesc(dwBiasesDims, biasPrc, memory::format::x); - cfg.inConfs.push_back(dataConfig); + dataConfig.desc = MKLDNNMemoryDesc(dwBiasesDims, biasPrc, memory::format::x); + cfg.inConfs.push_back(dataConfig); + } } for (size_t j = 0; j < desc.outputNumbers(); j++) { diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.h b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.h index ebb0e682fed..857cb66e70a 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.h +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.h @@ -51,6 +51,8 @@ private: mkldnn::memory::data_type dw_conv_in_dt = mkldnn::memory::data_type::data_undef; std::vector PostOpsIntBlobMemory; + int baseInputsNumber; + float pad_value = 0.f; }; diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/execution_graph_tests/num_inputs_fusing_bin_conv.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/execution_graph_tests/num_inputs_fusing_bin_conv.cpp new file mode 100644 index 00000000000..11c49f50259 --- /dev/null +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/execution_graph_tests/num_inputs_fusing_bin_conv.cpp @@ -0,0 +1,11 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "execution_graph_tests/num_inputs_fusing_bin_conv.hpp" +#include "common_test_utils/test_constants.hpp" + +using namespace LayerTestsDefinitions; + +INSTANTIATE_TEST_CASE_P(inputsNumFusingBinConv, ExecGraphInputsFusingBinConv, ::testing::Values(CommonTestUtils::DEVICE_CPU), + ExecGraphInputsFusingBinConv::getTestCaseName); diff --git a/inference-engine/tests/functional/plugin/shared/include/execution_graph_tests/num_inputs_fusing_bin_conv.hpp b/inference-engine/tests/functional/plugin/shared/include/execution_graph_tests/num_inputs_fusing_bin_conv.hpp new file mode 100644 index 00000000000..8a335f613eb --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/include/execution_graph_tests/num_inputs_fusing_bin_conv.hpp @@ -0,0 +1,25 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "ngraph_functions/builders.hpp" +#include "functional_test_utils/layer_test_utils.hpp" + +namespace LayerTestsDefinitions { + +class ExecGraphInputsFusingBinConv : public CommonTestUtils::TestsCommon, public testing::WithParamInterface { +public: + static std::string getTestCaseName(testing::TestParamInfo obj); + std::shared_ptr fnPtr; + std::string targetDevice; + +protected: + void SetUp() override; + void TearDown() override; +}; + +} // namespace LayerTestsDefinitions diff --git a/inference-engine/tests/functional/plugin/shared/src/execution_graph_tests/num_inputs_fusing_bin_conv.cpp b/inference-engine/tests/functional/plugin/shared/src/execution_graph_tests/num_inputs_fusing_bin_conv.cpp new file mode 100644 index 00000000000..e458828f27f --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/src/execution_graph_tests/num_inputs_fusing_bin_conv.cpp @@ -0,0 +1,72 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include + +#include "functional_test_utils/plugin_cache.hpp" +#include "functional_test_utils/layer_test_utils.hpp" + +#include "execution_graph_tests/num_inputs_fusing_bin_conv.hpp" + +#include "network_serializer.h" + +namespace LayerTestsDefinitions { + +std::string ExecGraphInputsFusingBinConv::getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice = obj.param; + return "targetDevice=" + targetDevice; +} + +void ExecGraphInputsFusingBinConv::SetUp() { + const InferenceEngine::SizeVector inputShapes = { 1, 16, 30, 30}, binConvKernelSize = {2, 2}, convKernelSize = {3, 3}; + const size_t numOutChannels = 16, numGroups = 16; + const std::vector strides = {1, 1}, dilations = {1, 1}; + const std::vector padsBegin = {1, 1}, padsEnd = {0, 0}; + const ngraph::op::PadType paddingType = ngraph::op::PadType::EXPLICIT; + const float padValue = 1.0; + targetDevice = this->GetParam(); + + auto params = ngraph::builder::makeParams(ngraph::element::f32, {inputShapes}); + auto binConv = ngraph::builder::makeBinaryConvolution(params[0], binConvKernelSize, strides, padsBegin, padsEnd, dilations, paddingType, numOutChannels, + padValue); + auto conv = ngraph::builder::makeGroupConvolution(binConv, ngraph::element::f32, convKernelSize, strides, padsBegin, padsEnd, dilations, paddingType, + numOutChannels, numGroups); + + auto biasNode = std::make_shared(ngraph::element::f32, std::vector{16, 1, 1}); + auto add = std::make_shared(conv, biasNode); + ngraph::ResultVector results{std::make_shared(add)}; + fnPtr = std::make_shared(results, params, "BinConvFuseConv"); +} + +void ExecGraphInputsFusingBinConv::TearDown() { + if (targetDevice.find(CommonTestUtils::DEVICE_GPU) != std::string::npos) { + PluginCache::get().reset(); + } +} + +TEST_P(ExecGraphInputsFusingBinConv, CheckNumInputsInBinConvFusingWithConv) { + InferenceEngine::CNNNetwork cnnNet(fnPtr); + auto ie = PluginCache::get().ie(); + auto execNet = ie->LoadNetwork(cnnNet, targetDevice); + + IE_SUPPRESS_DEPRECATED_START + InferenceEngine::CNNNetwork execGraphInfo = execNet.GetExecGraphInfo(); + std::vector nodes; + ASSERT_NO_THROW(nodes = InferenceEngine::Serialization::TopologicalSort(execGraphInfo)); + for (auto &node : nodes) { + if (node->type == "BinaryConvolution") { + std::string originalLayersNames = node->params["originalLayersNames"]; + ASSERT_TRUE(originalLayersNames.find("BinaryConvolution") != std::string::npos); + ASSERT_TRUE(originalLayersNames.find("Add") != std::string::npos); + ASSERT_EQ(node->insData.size(), 1); + } + } + IE_SUPPRESS_DEPRECATED_END + + fnPtr.reset(); +}; + +} // namespace LayerTestsDefinitions diff --git a/inference-engine/tests/ngraph_functions/include/ngraph_functions/builders.hpp b/inference-engine/tests/ngraph_functions/include/ngraph_functions/builders.hpp index b7821dc8225..0b585ae5c31 100644 --- a/inference-engine/tests/ngraph_functions/include/ngraph_functions/builders.hpp +++ b/inference-engine/tests/ngraph_functions/include/ngraph_functions/builders.hpp @@ -98,6 +98,17 @@ std::shared_ptr makeGroupConvolutionBackpropData(const ngraph::Out bool addBiases = false, const std::vector &biasesWeights = {}); +std::shared_ptr makeBinaryConvolution(const ngraph::Output &in, + const std::vector &filterSize, + const std::vector &strides, + const std::vector &padsBegin, + const std::vector &padsEnd, + const std::vector &dilations, + const op::PadType &autoPad, + size_t numOutChannels, + float padValue, + const std::vector &filterWeihgts = {}); + std::shared_ptr makeSplit(const ngraph::Output &in, const element::Type &type, size_t numSplits, diff --git a/inference-engine/tests/ngraph_functions/src/binary_convolution.cpp b/inference-engine/tests/ngraph_functions/src/binary_convolution.cpp new file mode 100644 index 00000000000..12e04d17ac4 --- /dev/null +++ b/inference-engine/tests/ngraph_functions/src/binary_convolution.cpp @@ -0,0 +1,44 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +#include "ngraph_functions/builders.hpp" +#include "ngraph_functions/utils/data_utils.hpp" + +namespace ngraph { +namespace builder { + +std::shared_ptr makeBinaryConvolution(const Output &in, + const std::vector &filterSize, + const std::vector &strides, + const std::vector &padsBegin, + const std::vector &padsEnd, + const std::vector &dilations, + const op::PadType &autoPad, + size_t numOutChannels, + float padValue, + const std::vector &filterWeihgts) { + auto shape = in.get_shape(); + std::vector filterWeightsShape = {numOutChannels, shape[1]}; + filterWeightsShape.insert(filterWeightsShape.end(), filterSize.begin(), filterSize.end()); + auto filterWeightsNode = std::make_shared(element::u1, filterWeightsShape); + size_t byteNum = ngraph::shape_size(filterWeightsShape) / sizeof(int8_t); + int8_t *buffer = const_cast(filterWeightsNode->get_data_ptr()); + if (filterWeihgts.size() == 0) { + std::vector weihgts = NGraphFunctions::Utils::generateVector(byteNum); + for (size_t i = 0; i < byteNum; i++) + buffer[i] = weihgts[i]; + } else { + for (size_t i = 0; i < byteNum; i++) + buffer[i] = filterWeihgts[i]; + } + auto conv = std::make_shared(in, filterWeightsNode, strides, padsBegin, padsEnd, dilations, + opset1::BinaryConvolution::BinaryConvolutionMode::XNOR_POPCOUNT, padValue, autoPad); + return conv; +} + +} // namespace builder +} // namespace ngraph \ No newline at end of file