diff --git a/inference-engine/thirdparty/clDNN/api/intel_gpu/graph/program.hpp b/inference-engine/thirdparty/clDNN/api/intel_gpu/graph/program.hpp index dcda8f1a7cd..f5ae92107f2 100644 --- a/inference-engine/thirdparty/clDNN/api/intel_gpu/graph/program.hpp +++ b/inference-engine/thirdparty/clDNN/api/intel_gpu/graph/program.hpp @@ -190,7 +190,9 @@ public: bool extract_and_remove(program_node& node); // Fuses two nodes into fused_node and removes peer_node from graph - void fuse_nodes(program_node& fused_node, program_node& peer_node, std::map>* fusing_history); + void fuse_nodes(program_node& fused_node, + program_node& peer_node, + std::map>>* fusing_history); // returns if 'node' has been removed bool remove_if_dangling(program_node& node); diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_1x1.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_1x1.cpp index b2a88d57ec3..0780442a6ec 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_1x1.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_1x1.cpp @@ -225,10 +225,12 @@ JitConstants BinaryConvolutionKernel1x1::GetFusedPrimitivesJitConstants(const bi eltwise_fused_ops += "\\\n\t" + data_type + " " + e_add + " = (oc < 16) ? " + get_shuffle(var_name + ".s0", "oc") + " : " + get_shuffle(var_name + ".s1", "oc") + ";"; eltwise_fused_ops += "\\\n\tres = res+" + e_add + ";"; - } else { + } else if (eltwise_p->mode == EltwiseMode::MUL) { eltwise_fused_ops += "\\\n\t" + data_type + " " + e_mul + " = (oc < 16) ? " + get_shuffle(var_name + ".s0", "oc") + " : " + get_shuffle(var_name + ".s1", "oc") + ";"; eltwise_fused_ops += "\\\n\tres = res*" + e_mul + ";"; + } else { + throw std::invalid_argument("Not supported eltwise fusing op in binary_convolution_1x1 kernel: " + params.layerID); } break; diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_1x1_b_fs_yx_fsv16.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_1x1_b_fs_yx_fsv16.cpp index fa0244d2dfa..a0073c198bd 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_1x1_b_fs_yx_fsv16.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_1x1_b_fs_yx_fsv16.cpp @@ -184,9 +184,11 @@ JitConstants BinaryConvolutionKernel1x1_b_fs_yx_fsv16::GetFusedPrimitivesJitCons if (eltwise_p->mode == EltwiseMode::ADD) { eltwise_fused_ops += "\\\n\t" + data_type + " " + e_add + " = " + get_shuffle(var_name, "oc") + ";"; eltwise_fused_ops += "\\\n\tres = res+" + var_name + ";"; - } else { + } else if (eltwise_p->mode == EltwiseMode::MUL) { eltwise_fused_ops += "\\\n\t" + data_type + " " + e_mul + " = " + get_shuffle(var_name, "oc") + ";"; eltwise_fused_ops += "\\\n\tres = res*" + var_name + ";"; + } else { + throw std::invalid_argument("Not supported eltwise fusing op in binary_convolution_1x1_fsv16 kernel: " + params.layerID); } break; diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_generic.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_generic.cpp index 879d49837e5..9c439b8aa8b 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_generic.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/actual_kernels/binary_convolution/binary_convolution_kernel_generic.cpp @@ -238,9 +238,11 @@ JitConstants BinaryConvolutionKernelGeneric::GetFusedPrimitivesJitConstants(cons if (eltwise_p->mode == EltwiseMode::ADD) { eltwise_fused_ops += data_type + " " + e_add + " = (i < 16) ? " + var_name + ".s0" + " : " + var_name + ".s1;"; eltwise_fused_ops += "res = res+" + e_add +";"; - } else { + } else if (eltwise_p->mode == EltwiseMode::MUL) { eltwise_fused_ops += data_type + " " + e_mul + " = (i < 16) ? " + var_name + ".s0" + " : " + var_name + ".s1;"; eltwise_fused_ops += "res = res*" + e_mul +";"; + } else { + throw std::invalid_argument("Not supported eltwise fusing op in binary_convolution_generic kernel: " + params.layerID); } break; diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp index 6ba250a1df0..e5927422532 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp @@ -1509,7 +1509,18 @@ JitConstants FusedOpsCodeGenerator::MakeOpJitConstants(const FusedOpsConfigurati auto idx = conf.bfzyx_idx_order; std::string shuffle_var = conf.shuffle_var_name; bool is_shuffled = false; - auto& fused_op_ids = desc.fused_op_ids; + + auto& dep_data = desc.dep_data; + int first_fused_ops_idx = -1; + size_t dep_idx = 0; + for (auto dep : dep_data) { + if (dep.dep_type == kernel_selector::DepType::INTERNAL) { + first_fused_ops_idx = static_cast(dep_idx); + break; + } + dep_idx++; + } + std::vector input_vars; out_var = GetOutputVarName(in_var, desc.op_id); @@ -1531,12 +1542,8 @@ JitConstants FusedOpsCodeGenerator::MakeOpJitConstants(const FusedOpsConfigurati auto get_acc_t = [&]() -> Datatype { std::vector input_types = {desc.output_tensor.GetDType()}; - for (auto& in : desc.tensors) { - input_types.push_back(in.GetDType()); - } - - for (auto& in : fused_op_ids) { - input_types.push_back(in.second); + for (auto& dep : dep_data) { + input_types.push_back(dep.data_type); } std::vector types_prioritized = { Datatype::F32, Datatype::F16 }; @@ -1551,15 +1558,11 @@ JitConstants FusedOpsCodeGenerator::MakeOpJitConstants(const FusedOpsConfigurati }; auto get_input = [&](size_t index) -> std::string { - auto input_name = in_var; - auto input_type = in_type; - if (index > 0) { - size_t input_idx = index - 1; - size_t tensors_len = desc.tensors.size(); - input_name = (input_idx < tensors_len)? GetInputVarName(input_idx, is_shuffled, shuffle_var) - : GetOutputVarName(in_var, fused_op_ids[input_idx - tensors_len].first); - input_type = (input_idx < tensors_len)? desc.tensors[input_idx].GetDType() : fused_op_ids[input_idx - tensors_len].second; - } + const auto dep = dep_data[index]; + auto input_name = (dep.dep_type == kernel_selector::DepType::ORIGINAL)? in_var : + (dep.dep_type == kernel_selector::DepType::INTERNAL)? GetOutputVarName(in_var, dep.op_id) + : GetInputVarName(dep.op_id, is_shuffled, shuffle_var); + auto input_type = (dep.dep_type == kernel_selector::DepType::ORIGINAL)? in_type : dep.data_type; auto acc_t = get_acc_t(); if (input_type != acc_t) @@ -1568,13 +1571,7 @@ JitConstants FusedOpsCodeGenerator::MakeOpJitConstants(const FusedOpsConfigurati return input_name; }; - // Generate input variable list: dst + tensor inputs + fused ops input - // If the input_vars_length are larger than max_num_input_vars, do not add dst to input variable list. - // because dst is not used, when Fused op has both tensor and fused input. - size_t input_vars_length = 1 + desc.tensors.size() + fused_op_ids.size(); // dst + tensor inputs + fused ops input - size_t max_num_input_vars = (desc.tensors.size() > 1)? 3 : 2; - size_t start_idx = (input_vars_length > max_num_input_vars) ? 1 : 0; - for (size_t i = start_idx; i < input_vars_length; i++) { + for (size_t i = 0; i < dep_data.size(); i++) { input_vars.push_back(get_input(i)); } @@ -1595,7 +1592,6 @@ JitConstants FusedOpsCodeGenerator::MakeOpJitConstants(const FusedOpsConfigurati auto p = desc.GetOpParams(); if (!p) throw std::runtime_error("[clDNN] Eltwise fuse params can't be nullptr"); - std::string op = ""; switch (p->mode) { case kernel_selector::EltwiseMode::ADD: @@ -1604,6 +1600,12 @@ JitConstants FusedOpsCodeGenerator::MakeOpJitConstants(const FusedOpsConfigurati case kernel_selector::EltwiseMode::MUL: op = "*"; break; + case kernel_selector::EltwiseMode::SUB: + op = "-"; + break; + case kernel_selector::EltwiseMode::DIV: + op = "/"; + break; default: throw std::runtime_error("[clDNN] Eltwise mode is not supported in fused ops codegen"); } @@ -1618,8 +1620,8 @@ JitConstants FusedOpsCodeGenerator::MakeOpJitConstants(const FusedOpsConfigurati if (!p) throw std::runtime_error("[clDNN] Quantize fuse params can't be nullptr"); - std::string in_converted = (fused_op_ids.empty()) ? in_var : GetOutputVarName(in_var, fused_op_ids[0].first); - Datatype input_type = (fused_op_ids.empty()) ? in_type : fused_op_ids[0].second; + std::string in_converted = (first_fused_ops_idx < 0) ? in_var : GetOutputVarName(in_var, dep_data[first_fused_ops_idx].op_id); + Datatype input_type = (first_fused_ops_idx < 0) ? in_type : dep_data[first_fused_ops_idx].data_type; Datatype tmp_type = Datatype::F32; std::string tmp_type_str = GetType(tmp_type, vec_size); std::string tmp_var = out_var + "_tmp"; @@ -1722,8 +1724,7 @@ JitConstants FusedOpsCodeGenerator::MakeOpJitConstants(const FusedOpsConfigurati case KernelType::ACTIVATION: { auto p = desc.GetOpParams(); base_activation_params activation_p = p->param; - - std::string new_in_var = (fused_op_ids.empty()) ? in_var : GetOutputVarName(in_var, fused_op_ids[0].first); + std::string new_in_var = (first_fused_ops_idx < 0) ? in_var : GetOutputVarName(in_var, dep_data[first_fused_ops_idx].op_id); op_decls += "\\\n\t" + GetOutputType(vec_size) + " " + out_var + " = " + ConvertToOutputType(new_in_var, vec_size) + ";"; if (activation_p.function != ActivationFunction::NONE) { auto suffix = "_FUSED_OP"+toCodeString(desc.op_id) + conf.suffix; diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector.cpp index 2105b043658..c71561bc48c 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector.cpp @@ -14,6 +14,7 @@ #include #include #include +#include "intel_gpu/runtime/debug_configuration.hpp" // #define ENABLE_ENV // #define ENABLE_ENV_PRINT @@ -98,8 +99,14 @@ KernelsData kernel_selector_base::GetNaiveBestKernel(const Params& params, } #endif } - } catch (std::runtime_error&) { + } catch (std::runtime_error& ex) { // we have to handle it in order to avoid exception in KernelSelector as much we can + GPU_DEBUG_GET_INSTANCE(debug_config); + GPU_DEBUG_IF(debug_config->verbose >= 3) { + kernelName = (implementation != nullptr)? implementation->GetName() : "[impl is null]"; + GPU_DEBUG_COUT << "layerID: " << params.layerID << " kenrel: " + << kernelName << " - " << ex.what() << std::endl; + } } } @@ -187,8 +194,14 @@ KernelsData kernel_selector_base::GetAutoTuneBestKernel(const Params& params, kernelName = implementation->GetName(); } } - } catch (std::runtime_error&) { + } catch (std::runtime_error& ex) { // we have to handle it in order to avoid exception in KernelSelector as much we can + GPU_DEBUG_GET_INSTANCE(debug_config); + GPU_DEBUG_IF(debug_config->verbose >= 3) { + kernelName = (implementation != nullptr)? implementation->GetName() : "[impl is null]"; + GPU_DEBUG_COUT << "layerID: " << params.layerID << " kenrel: " + << kernelName << " - " << ex.what() << std::endl; + } } } } @@ -210,8 +223,14 @@ KernelsData kernel_selector_base::GetAutoTuneBestKernel(const Params& params, kernelName = implementation->GetName(); } } - } catch (std::runtime_error&) { + } catch (std::runtime_error& ex) { // we have to handle it in order to avoid exception in KernelSelector as much we can + GPU_DEBUG_GET_INSTANCE(debug_config); + GPU_DEBUG_IF(debug_config->verbose >= 3) { + kernelName = (implementation != nullptr)? implementation->GetName() : "[impl is null]"; + GPU_DEBUG_COUT << "layerID: " << params.layerID << " kenrel: " + << kernelName << " - " << ex.what() << std::endl; + } } } } diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_params.h b/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_params.h index 683a8a78da9..06bc8435ca7 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_params.h +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_params.h @@ -490,6 +490,26 @@ struct FusedOpsConfiguration { bool IsPostReorderFused(void) const { return orig_output_layout != DataLayout::DataLayoutCount; } }; +// Dependency(Input) type of fusing operation in fused node. +// There are different ways to generate input var name and type by the dependency(input) type in MakeOpJitConstants in jitter +// - ORIGINAL: The input of the operation is the fused node such as Conv +// - EXTERNAL: The input of the operation is the external node outside the fused node +// - INTERNAL: The input of the operation is the another fused operation in the fused node +enum class DepType { + UNDEFINED = -1, + ORIGINAL = 0, + EXTERNAL = 1, + INTERNAL = 2 +}; + +// Dependency(Input) information of fusing operation which is used to generate input var name and type +// in MakeOpJitConstants in jitter +struct dep_info { + DepType dep_type = DepType::UNDEFINED; + size_t op_id; + Datatype data_type; +}; + // Instance of fused_operation_desc is added to fused_ops vector if a node has been fused to current one using program::fuse_nodes // method. In order to process fused ops following modifications should be done in a kernel: // option 1 - using common generator: @@ -542,7 +562,7 @@ struct fused_operation_desc { MultiDataTensor tensors; DataTensor output_tensor; size_t op_id; - std::vector> fused_op_ids; + std::vector dep_data = {}; // Helper functions for operation generation KernelType GetType() const { return op_params->GetType(); } diff --git a/inference-engine/thirdparty/clDNN/src/graph_optimizer/prepare_primitive_fusing.cpp b/inference-engine/thirdparty/clDNN/src/graph_optimizer/prepare_primitive_fusing.cpp index ae3f15469f4..5c2dce910b2 100644 --- a/inference-engine/thirdparty/clDNN/src/graph_optimizer/prepare_primitive_fusing.cpp +++ b/inference-engine/thirdparty/clDNN/src/graph_optimizer/prepare_primitive_fusing.cpp @@ -185,7 +185,7 @@ void prepare_primitive_fusing::fuse_reorders(program &p) { void prepare_primitive_fusing::fuse_activations(program &p) { bool is_debug = p.get_options().get()->enabled(); - std::map> fusing_history; + std::map>> fusing_history; bool use_onednn_impls = false; #ifdef ENABLE_ONEDNN_FOR_GPU @@ -457,7 +457,7 @@ void prepare_primitive_fusing::fuse_bias(program &p) { void prepare_primitive_fusing::fuse_simple_primitives(program &p) { bool recalc_processing_order = false; - std::map> fusing_history; + std::map>> fusing_history; const uint8_t supports_immad = p.get_engine().get_device_info().supports_immad; auto itr = p.get_processing_order().begin(); @@ -631,13 +631,15 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { return true; }; - auto get_users_from_fusing_history = [&](primitive_id id) { + auto get_users_from_fusing_history = [&](const primitive_id& id) { std::vector users; - for (auto deps_data : fusing_history) { - auto key = deps_data.first; - auto deps_vec = deps_data.second; - auto iter = std::find(deps_vec.begin(), deps_vec.end(), id); - if (iter != deps_vec.end()) { + for (auto fusing_info : fusing_history) { + auto key = fusing_info.first; + auto dep_info_vec = fusing_info.second; + auto iter = std::find_if(dep_info_vec.begin(), dep_info_vec.end(), [&](std::pair& dep_info) { + return (id == dep_info.first); + }); + if (iter != dep_info_vec.end()) { users.push_back(key); } } @@ -660,7 +662,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { auto& fused_descs = input_data.get_fused_primitives(); auto origin_input_iter = std::find_if(fused_descs.begin(), fused_descs.end(), [&](cldnn::fused_primitive_desc& desc) { - return (desc.node->id() == prim_id); + return (desc.node->id() == prim_id.first); }); if (origin_input_iter != fused_descs.end()) { auto users = get_users_from_fusing_history(origin_input_iter->node->id()); @@ -922,7 +924,9 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { std::shared_ptr prim = node.get_primitive(); const std::vector supported_modes = { eltwise_mode::sum, - eltwise_mode::prod + eltwise_mode::prod, + eltwise_mode::sub, + eltwise_mode::div }; if (node.is_output() || node.inputs_count() != 2 || @@ -937,7 +941,8 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { for (size_t i = 0; i < parents.size(); i++) { can_fuse_parents[i] = (parents[i]->is_type() && conv_supports_fusings(parents[i]->as())) || - (parents[i]->is_type() && bin_conv_supports_eltw_fusings(parents[i]->as())) || + ((prim->mode == eltwise_mode::sum || prim->mode == eltwise_mode::prod) && + ((parents[i]->is_type() && bin_conv_supports_eltw_fusings(parents[i]->as())) || (parents[i]->is_type() && mvn_supports_fusings(parents[i]->as())) || (parents[i]->is_type()) || (parents[i]->is_type()) || @@ -955,7 +960,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { (parents[i]->is_type()) || (parents[i]->is_type() && pooling_supports_fusings(parents[i]->as())) || (parents[i]->is_type() && dts_supports_fusings(parents[i]->as())) || - (parents[i]->is_type() && reduce_supports_fusings(parents[i]->as())); + (parents[i]->is_type() && reduce_supports_fusings(parents[i]->as())))); } // Disable fusion to a node on constant path when second input is in data flow @@ -1096,12 +1101,11 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { } while (node_queue.size() > 1); } else { merge_allowed = fused_node->get_users().size() == 1; + for (auto& parent : fused_node->get_dependencies()) + if (parent->id() == peer_node->id()) + merge_allowed = false; } - for (auto& parent : fused_node->get_dependencies()) - if (parent->id() == peer_node->id()) - merge_allowed = false; - if (!merge_allowed) return; @@ -1150,13 +1154,10 @@ void prepare_primitive_fusing::optimize_fused_ops(program& p) { if (desc.node->id() == prim.node->id()) { continue; } - - auto rm_iter = std::find_if(prim.fused_deps.begin(), prim.fused_deps.end(), [&](primitive_id& dep_id){ - return (desc.node->id() == dep_id); - }); + auto rm_iter = prim.fused_deps.find(desc.node->id()); if (rm_iter != prim.fused_deps.end()) { prim.fused_deps.erase(rm_iter); - prim.fused_deps.insert(prim.fused_deps.end(), desc.fused_deps.begin(), desc.fused_deps.end()); + prim.fused_deps.insert(desc.fused_deps.begin(), desc.fused_deps.end()); } } }; diff --git a/inference-engine/thirdparty/clDNN/src/include/kernel_selector_helper.h b/inference-engine/thirdparty/clDNN/src/include/kernel_selector_helper.h index 873709a69cc..f6ef1590843 100644 --- a/inference-engine/thirdparty/clDNN/src/include/kernel_selector_helper.h +++ b/inference-engine/thirdparty/clDNN/src/include/kernel_selector_helper.h @@ -160,7 +160,7 @@ inline params_t get_default_params(const arg_t& arg, uint32_t split = 1) { params.layerID = arg.id(); convert_fused_activation_func_params(arg, params.activations); - std::map> prim_op_id_map; + std::map> prim_id_type_map; size_t op_id = 0; for (auto& fused_prim : arg.get_fused_primitives()) { kernel_selector::fused_operation_desc desc; @@ -169,22 +169,43 @@ inline params_t get_default_params(const arg_t& arg, uint32_t split = 1) { CLDNN_ERROR_MESSAGE(arg.id(), "Invalid fused operation (" + fused_prim.node->id() + ") of type " + fused_prim.node->get_primitive()->type_string() ); } + desc.dep_idx_start = fused_prim.dep_start_idx; desc.dep_size = fused_prim.deps.size(); desc.op_id = op_id++; desc.output_tensor = convert_data_tensor(fused_prim.output_layout); - prim_op_id_map[fused_prim.node->id()] = std::make_pair(desc.op_id, desc.output_tensor.GetDType()); - for (auto& dep : fused_prim.fused_deps) { - auto iter = prim_op_id_map.find(dep); - if (iter != prim_op_id_map.end()) { - desc.fused_op_ids.push_back(iter->second); - } - } + prim_id_type_map[fused_prim.node->id()] = std::make_pair(desc.op_id, desc.output_tensor.GetDType()); for (size_t i = desc.dep_idx_start; i < desc.dep_idx_start + desc.dep_size; i++) { desc.tensors.push_back(convert_data_tensor(arg.get_dependency(i).get_output_layout())); } + if (fused_prim.total_num_deps > 0) { + desc.dep_data.resize(fused_prim.total_num_deps); + for (auto& dep : fused_prim.fused_deps) { + auto iter = prim_id_type_map.find(dep.first); + if (iter != prim_id_type_map.end()) { + auto& op_data = iter->second; + desc.dep_data[dep.second].dep_type = kernel_selector::DepType::INTERNAL; + desc.dep_data[dep.second].op_id = op_data.first; + desc.dep_data[dep.second].data_type = op_data.second; + } + } + + int idx = 0; + for (auto& dep : fused_prim.deps) { + desc.dep_data[dep.second].dep_type = kernel_selector::DepType::EXTERNAL; + desc.dep_data[dep.second].op_id = idx; + desc.dep_data[dep.second].data_type = desc.tensors[idx++].GetDType(); + } + + for (auto& dep : desc.dep_data) { + if (dep.dep_type == kernel_selector::DepType::UNDEFINED) { + dep.dep_type = kernel_selector::DepType::ORIGINAL; + break; + } + } + } params.fused_ops.push_back(desc); } diff --git a/inference-engine/thirdparty/clDNN/src/include/program_node.h b/inference-engine/thirdparty/clDNN/src/include/program_node.h index 27f425fd7de..a892318ca82 100644 --- a/inference-engine/thirdparty/clDNN/src/include/program_node.h +++ b/inference-engine/thirdparty/clDNN/src/include/program_node.h @@ -60,8 +60,9 @@ struct fused_primitive_desc_onednn { struct fused_primitive_desc { std::shared_ptr node; size_t dep_start_idx; - std::vector deps; - std::vector fused_deps; + std::map deps; + std::map fused_deps; + size_t total_num_deps = 0; activation_func activation; activation_additional_params activation_params; layout input_layout = layout(data_types::f32, format::bfyx, tensor()); diff --git a/inference-engine/thirdparty/clDNN/src/program.cpp b/inference-engine/thirdparty/clDNN/src/program.cpp index 9a4bd27cc6b..9d60731d1e2 100644 --- a/inference-engine/thirdparty/clDNN/src/program.cpp +++ b/inference-engine/thirdparty/clDNN/src/program.cpp @@ -993,11 +993,15 @@ bool program::extract_and_remove(program_node& node) { return true; } -void program::fuse_nodes(program_node &fused_node, program_node &peer_node, std::map>* fusing_history) { + +void program::fuse_nodes(program_node &fused_node, + program_node &peer_node, + std::map>>* fusing_history) { auto peer_layout = peer_node.get_output_layout(); fused_primitive_desc local_desc; local_desc.node = get_node_ptr(peer_node.id()); local_desc.dep_start_idx = fused_node.get_dependencies().size(); + local_desc.total_num_deps = peer_node.get_dependencies().size(); local_desc.output_layout = peer_layout; local_desc.activation = activation_func::none; if (!peer_node.get_fused_activations_funcs().empty()) { @@ -1014,15 +1018,18 @@ void program::fuse_nodes(program_node &fused_node, program_node &peer_node, std: auto history_iter = fusing_history->find(peer_node.id()); if (history_iter != fusing_history->end()) { for (auto& id : history_iter->second) { - local_desc.fused_deps.push_back(id); + local_desc.fused_deps.emplace(id.first, id.second); } } // Add new dependencies to the fused_node + size_t deps_idx = 0; for (size_t i = 0; i < peer_node.get_dependencies().size(); i++) { auto& dep = peer_node.get_dependency(i); - if (dep.id() == fused_node.id()) + if (dep.id() == fused_node.id()) { + deps_idx++; continue; + } if (peer_node.is_type()) { quantize_node& q_node = peer_node.as(); @@ -1048,9 +1055,11 @@ void program::fuse_nodes(program_node &fused_node, program_node &peer_node, std: } } fused_node.dependencies.push_back(&dep); - local_desc.deps.push_back(dep.id()); + local_desc.deps.emplace(dep.id(), deps_idx++); dep.users.push_back(&fused_node); } + local_desc.total_num_deps = std::min(local_desc.total_num_deps, deps_idx); + fused_node.add_fused_primitive(local_desc); // This shouldn't happen, but who knows... if (peer_node.has_fused_primitives()) { @@ -1059,7 +1068,13 @@ void program::fuse_nodes(program_node &fused_node, program_node &peer_node, std: add_optimized_primitive_info(peer_node.id(), { fused_node.id() }); for (auto& user : peer_node.users) { - (*fusing_history)[user->id()].push_back(peer_node.id()); + size_t dep_idx = 0; + for (auto& dep : user->dependencies) { + if (dep->id() == peer_node.id()) + break; + dep_idx++; + } + (*fusing_history)[user->id()].push_back(std::make_pair(peer_node.id(), dep_idx)); } // Remove all edges connected with peer node diff --git a/inference-engine/thirdparty/clDNN/src/program_node.cpp b/inference-engine/thirdparty/clDNN/src/program_node.cpp index 56bac3b47e3..a97d881a485 100644 --- a/inference-engine/thirdparty/clDNN/src/program_node.cpp +++ b/inference-engine/thirdparty/clDNN/src/program_node.cpp @@ -107,7 +107,11 @@ std::unique_ptr program_node::desc_to_json() const { for (auto& fused_desc : get_fused_primitives()) { json_composite fused_node_info; fused_node_info.add("id", fused_desc.node->id()); - fused_node_info.add("dependencies", fused_desc.deps); + std::vector dep_ids; + for (auto dep : fused_desc.deps) { + dep_ids.push_back(dep.first); + } + fused_node_info.add("dependencies", dep_ids); fused_node_info.add("dep start_idx", fused_desc.dep_start_idx); json_composite info; info.add("data type", dt_to_str(fused_desc.output_layout.data_type)); diff --git a/inference-engine/thirdparty/clDNN/tests/test_cases/fusings_gpu_test.cpp b/inference-engine/thirdparty/clDNN/tests/test_cases/fusings_gpu_test.cpp index 19ca152ff88..6b5743b32e3 100644 --- a/inference-engine/thirdparty/clDNN/tests/test_cases/fusings_gpu_test.cpp +++ b/inference-engine/thirdparty/clDNN/tests/test_cases/fusings_gpu_test.cpp @@ -1052,6 +1052,138 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_multi_eltwise_4_clamp, bc_test_params{CASE_CONV_FP16_4, 2, 7}, })); +class conv_fp32_eltwise_fusing_extend_ops : public ConvFusingTest {}; +TEST_P(conv_fp32_eltwise_fusing_extend_ops, pattern01_simple_sub) { + if (engine.get_device_info().supports_immad) { + return; + } + + auto p = GetParam(); + create_topologies(input_layout("input", get_input_layout(p)), + data("eltwise_data1", get_mem(get_output_layout(p))), + data("eltwise_data2", get_mem(get_output_layout(p))), + data("eltwise_data4", get_mem(get_output_layout(p))), + data("bias", get_mem(get_bias_layout(p))), + data("weights", get_mem(get_weights_layout(p))), + convolution("conv_prim", "input", { "weights" }, {"bias"}, p.groups, p.stride, p.pad, p.dilation), + eltwise("eltwise1_sum", "conv_prim", "eltwise_data1", eltwise_mode::sum), + eltwise("eltwise2_sub", "conv_prim", "eltwise_data2", eltwise_mode::sub), + eltwise("eltwise3_prod", "eltwise1_sum", "eltwise2_sub", eltwise_mode::prod), + eltwise("eltwise4_sum", "eltwise3_prod", "eltwise_data4", eltwise_mode::sum), + concatenation("concat", { "eltwise4_sum", "eltwise4_sum" }, cldnn::concatenation::along_f), + reorder("reorder_bfyx", "concat", p.default_format, data_types::f32) + ); + implementation_desc conv_impl = { format::b_fs_yx_fsv16, ""}; + bo_fused.set_option(build_option::force_implementations({ {"conv_prim", conv_impl} })); + + tolerance = 1e-5f; + execute(p); +} + +TEST_P(conv_fp32_eltwise_fusing_extend_ops, pattern02_sub_scale) { + if (engine.get_device_info().supports_immad) { + return; + } + + auto p = GetParam(); + create_topologies(input_layout("input", get_input_layout(p)), + data("eltwise_data1", get_mem(get_output_layout(p))), + data("eltwise_data2", get_mem(get_output_layout(p))), + data("scale_data", get_mem(get_per_channel_layout(p), 1.0f/p.kernel.count())), + data("bias", get_mem(get_bias_layout(p))), + data("weights", get_mem(get_weights_layout(p))), + convolution("conv_prim", "input", { "weights" }, {"bias"}, p.groups, p.stride, p.pad, p.dilation), + eltwise("eltwise1_sum", "conv_prim", "eltwise_data1", eltwise_mode::sum), + eltwise("eltwise2_sub", "conv_prim", "eltwise1_sum", eltwise_mode::sub), + eltwise("eltwise3_prod", "eltwise2_sub", "eltwise_data2", eltwise_mode::prod), + scale("scale", "eltwise3_prod", "scale_data"), + concatenation("concat", { "scale", "scale" }, cldnn::concatenation::along_f), + reorder("reorder_bfyx", "concat", p.default_format, data_types::f32) + ); + implementation_desc conv_impl = { format::b_fs_yx_fsv16, ""}; + bo_fused.set_option(build_option::force_implementations({ {"conv_prim", conv_impl} })); + + tolerance = 1e-5f; + execute(p); +} + +TEST_P(conv_fp32_eltwise_fusing_extend_ops, pattern03_sub_div) { + if (engine.get_device_info().supports_immad) { + return; + } + + auto p = GetParam(); + create_topologies(input_layout("input", get_input_layout(p)), + data("eltwise_data1", get_mem(get_output_layout(p))), + data("eltwise_data2", get_mem(get_output_layout(p), 1.0f)), + data("eltwise_data3", get_mem(get_output_layout(p))), + data("eltwise_data4", get_mem(get_output_layout(p))), + data("bias", get_mem(get_bias_layout(p))), + data("weights", get_mem(get_weights_layout(p))), + convolution("conv_prim", "input", { "weights" }, {"bias"}, p.groups, p.stride, p.pad, p.dilation), + eltwise("eltwise1_sum", "conv_prim", "eltwise_data1", eltwise_mode::sum), + eltwise("eltwise2_div", "eltwise1_sum", "eltwise_data2", eltwise_mode::div), + eltwise("eltwise3_prod", "eltwise2_div", "eltwise_data3", eltwise_mode::prod), + eltwise("eltwise4_sum", "eltwise3_prod", "eltwise_data4", eltwise_mode::sum), + concatenation("concat", { "eltwise4_sum", "eltwise4_sum" }, cldnn::concatenation::along_f), + reorder("reorder_bfyx", "concat", p.default_format, data_types::f32) + ); + implementation_desc conv_impl = { format::b_fs_yx_fsv16, ""}; + bo_fused.set_option(build_option::force_implementations({ {"conv_prim", conv_impl} })); + + tolerance = 1e-5f; + execute(p); +} + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_eltwise_fusing_extend_ops, + ::testing::ValuesIn(std::vector{ + bc_test_params{CASE_CONV_FP32_2, 3, 7}, + bc_test_params{CASE_CONV_FP32_3, 3, 7}, + bc_test_params{CASE_CONV_FP32_4, 3, 7}, + + bc_test_params{CASE_CONV_FP16_2, 3, 7}, + bc_test_params{CASE_CONV_FP16_3, 3, 7}, + bc_test_params{CASE_CONV_FP16_4, 3, 7}, + })); + +class conv_fp32_eltwise_fusing_2conv : public ConvFusingTest {}; +TEST_P(conv_fp32_eltwise_fusing_2conv, basic) { + if (engine.get_device_info().supports_immad) { + return; + } + + auto p = GetParam(); + create_topologies(input_layout("input", get_input_layout(p)), + data("bias0", get_mem(get_bias_layout(p))), + data("weights0", get_mem(get_weights_layout(p))), + data("bias", get_mem(get_bias_layout(p))), + data("weights", get_mem(get_weights_layout(p))), + convolution("conv_prim0", "input", { "weights0" }, {"bias0"}, p.groups, p.stride, p.pad, p.dilation), + convolution("conv_prim", "input", { "weights" }, {"bias"}, p.groups, p.stride, p.pad, p.dilation), + eltwise("eltwise1", "conv_prim0", "conv_prim", eltwise_mode::sum), + eltwise("eltwise2", "conv_prim0", "conv_prim", eltwise_mode::sum), + eltwise("eltwise3", "eltwise1", "eltwise2", eltwise_mode::prod), + concatenation("concat", { "eltwise3", "eltwise3" }, cldnn::concatenation::along_f), + reorder("reorder_bfyx", "concat", p.default_format, data_types::f32) + ); + implementation_desc conv_impl = { format::b_fs_yx_fsv16, ""}; + bo_fused.set_option(build_option::force_implementations({ {"conv_prim0", conv_impl}, {"conv_prim", conv_impl} })); + + tolerance = 1e-5f; + execute(p); +} + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_eltwise_fusing_2conv, + ::testing::ValuesIn(std::vector{ + bc_test_params{CASE_CONV_FP32_2, 4, 7}, + bc_test_params{CASE_CONV_FP32_3, 4, 7}, + bc_test_params{CASE_CONV_FP32_4, 4, 7}, + + bc_test_params{CASE_CONV_FP16_2, 4, 7}, + bc_test_params{CASE_CONV_FP16_3, 4, 7}, + bc_test_params{CASE_CONV_FP16_4, 4, 7}, + })); + class conv_fp32_multi_eltwise_3_fusing : public ConvFusingTest {}; TEST_P(conv_fp32_multi_eltwise_3_fusing, basic) {