From fa9434494faf6ca4435f697242c2ffb432bacea9 Mon Sep 17 00:00:00 2001 From: Ivan Novoselov Date: Mon, 20 Dec 2021 22:45:01 +0300 Subject: [PATCH] [Snippets][CPU] Eltwise subgraphs code generation (#7936) --- .../src/mkldnn_plugin/CMakeLists.txt | 4 +- inference-engine/src/mkldnn_plugin/config.cpp | 3 + inference-engine/src/mkldnn_plugin/config.h | 1 + .../src/mkldnn_plugin/cpu_types.cpp | 3 + .../src/mkldnn_plugin/cpu_types.h | 3 +- .../mkldnn_plugin/emitters/cpu_generator.cpp | 139 ++++ .../mkldnn_plugin/emitters/cpu_generator.hpp | 33 + .../emitters/jit_eltwise_emitters.cpp | 20 +- .../emitters/jit_eltwise_emitters.hpp | 3 + .../mkldnn_plugin/emitters/jit_emitter.cpp | 6 +- .../mkldnn_plugin/emitters/jit_emitter.hpp | 16 +- .../emitters/jit_mkldnn_ext_emitters.hpp | 144 ++++ .../emitters/jit_snippets_emitters.hpp | 641 ++++++++++++++++++ .../mkldnn_plugin/mkldnn_nodes_factory.cpp | 2 + .../src/mkldnn_plugin/mkldnn_plugin.cpp | 56 +- .../convert_to_cpu_specific_opset.hpp | 2 - .../snippets_mark_skipped.cpp | 352 ++++++++++ .../snippets_mark_skipped.hpp | 39 ++ .../src/mkldnn_plugin/nodes/subgraph.cpp | 548 +++++++++++++++ .../src/mkldnn_plugin/nodes/subgraph.h | 91 +++ .../python/tests/test_onnx/test_zoo_models.py | 3 +- src/common/snippets/CMakeLists.txt | 10 +- .../snippets/include/snippets/generator.hpp | 6 +- .../{src => include/snippets}/itt.hpp | 2 +- .../snippets/include/snippets/op/kernel.hpp | 3 +- .../include/snippets/op/powerstatic.hpp | 44 ++ .../include/snippets/op/staticpower.hpp | 42 -- .../snippets/include/snippets/op/subgraph.hpp | 34 +- .../snippets/include/snippets/op/tile.hpp | 1 + .../snippets/pass/assign_registers.hpp | 2 +- .../snippets/pass/collapse_subgraph.hpp | 40 +- .../snippets/pass/insert_load_store.hpp | 4 +- .../snippets/pass/insert_movebroadcast.hpp | 2 +- .../load_movebroadcast_to_broadcastload.hpp | 2 +- .../snippets/pass/vector_to_scalar.hpp | 8 +- .../include/snippets/snippets_isa.hpp | 4 +- src/common/snippets/src/generator.cpp | 76 ++- src/common/snippets/src/op/broadcastload.cpp | 2 +- src/common/snippets/src/op/broadcastmove.cpp | 2 +- src/common/snippets/src/op/load.cpp | 2 +- src/common/snippets/src/op/store.cpp | 2 +- src/common/snippets/src/op/subgraph.cpp | 107 +-- .../snippets/src/pass/assign_registers.cpp | 7 +- .../snippets/src/pass/collapse_subgraph.cpp | 602 ++++++++-------- .../snippets/src/pass/insert_load_store.cpp | 4 +- .../src/pass/insert_movebroadcast.cpp | 9 +- .../load_movebroadcast_to_broadcastload.cpp | 7 +- .../snippets/src/pass/vector_to_scalar.cpp | 4 +- src/common/snippets/src/remarks.hpp | 4 +- .../inference_engine/skip_tests_config.cpp | 4 + .../snippets/tokenization.cpp | 291 +++++--- src/tests/unit/cpu/CMakeLists.txt | 7 + src/tests/unit/cpu/generate_add.cpp | 427 ++++++++++++ 53 files changed, 3265 insertions(+), 605 deletions(-) create mode 100644 inference-engine/src/mkldnn_plugin/emitters/cpu_generator.cpp create mode 100644 inference-engine/src/mkldnn_plugin/emitters/cpu_generator.hpp create mode 100644 inference-engine/src/mkldnn_plugin/emitters/jit_mkldnn_ext_emitters.hpp create mode 100644 inference-engine/src/mkldnn_plugin/emitters/jit_snippets_emitters.hpp create mode 100644 inference-engine/src/mkldnn_plugin/ngraph_transformations/snippets_mark_skipped.cpp create mode 100644 inference-engine/src/mkldnn_plugin/ngraph_transformations/snippets_mark_skipped.hpp create mode 100644 inference-engine/src/mkldnn_plugin/nodes/subgraph.cpp create mode 100644 inference-engine/src/mkldnn_plugin/nodes/subgraph.h rename src/common/snippets/{src => include/snippets}/itt.hpp (96%) create mode 100644 src/common/snippets/include/snippets/op/powerstatic.hpp delete mode 100644 src/common/snippets/include/snippets/op/staticpower.hpp create mode 100644 src/tests/unit/cpu/generate_add.cpp diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt index 24359ffe7f8..e6397b6a77c 100644 --- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt +++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt @@ -44,7 +44,8 @@ target_link_libraries(${TARGET_NAME} PRIVATE mkldnn inference_engine inference_engine_transformations inference_engine_lp_transformations - ov_shape_inference) + ov_shape_inference + inference_engine_snippets) target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_INFERENCE_EXTENSION_API) target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) @@ -75,6 +76,7 @@ target_include_directories(${TARGET_NAME}_obj PRIVATE $ $ $ + $ PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} $) diff --git a/inference-engine/src/mkldnn_plugin/config.cpp b/inference-engine/src/mkldnn_plugin/config.cpp index bec2c5ee0b6..02d12b349e2 100644 --- a/inference-engine/src/mkldnn_plugin/config.cpp +++ b/inference-engine/src/mkldnn_plugin/config.cpp @@ -115,6 +115,8 @@ void Config::readProperties(const std::map &prop) { IE_THROW() << "Wrong value for property key " << PluginConfigParams::KEY_ENFORCE_BF16 << ". Expected only YES/NO"; } + } else if (key == PluginConfigParams::KEY_CACHE_DIR) { + cache_dir = val; } else { IE_THROW(NotFound) << "Unsupported property " << key << " by CPU plugin"; } @@ -167,6 +169,7 @@ void Config::updateProperties() { _config.insert({ PluginConfigParams::KEY_PERFORMANCE_HINT, perfHintsConfig.ovPerfHint }); _config.insert({ PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, std::to_string(perfHintsConfig.ovPerfHintNumRequests) }); + _config.insert({PluginConfigParams::KEY_CACHE_DIR, cache_dir}); } } diff --git a/inference-engine/src/mkldnn_plugin/config.h b/inference-engine/src/mkldnn_plugin/config.h index c1edd197639..6a1e785866c 100644 --- a/inference-engine/src/mkldnn_plugin/config.h +++ b/inference-engine/src/mkldnn_plugin/config.h @@ -41,6 +41,7 @@ struct Config { #ifdef CPU_DEBUG_CAPS DebugCaps::Config debugCaps; #endif + std::string cache_dir{}; void readProperties(const std::map &config); void updateProperties(); diff --git a/inference-engine/src/mkldnn_plugin/cpu_types.cpp b/inference-engine/src/mkldnn_plugin/cpu_types.cpp index 9836b1a5f32..9facd0d07e8 100644 --- a/inference-engine/src/mkldnn_plugin/cpu_types.cpp +++ b/inference-engine/src/mkldnn_plugin/cpu_types.cpp @@ -183,6 +183,7 @@ const InferenceEngine::details::caseless_unordered_map type_t { "MatrixNms", MatrixNms}, { "MulticlassNms", MulticlassNms}, { "Reference", Reference}, + { "Subgraph", Subgraph} }; Type TypeFromName(const std::string& type) { @@ -364,6 +365,8 @@ std::string NameFromType(const Type type) { return "MulticlassNms"; case Reference: return "Reference"; + case Subgraph: + return "Subgraph"; default: return "Unknown"; } diff --git a/inference-engine/src/mkldnn_plugin/cpu_types.h b/inference-engine/src/mkldnn_plugin/cpu_types.h index 8052b73fe9d..9299febc116 100644 --- a/inference-engine/src/mkldnn_plugin/cpu_types.h +++ b/inference-engine/src/mkldnn_plugin/cpu_types.h @@ -99,7 +99,8 @@ enum Type { ExtractImagePatches, NonMaxSuppression, MatrixNms, - MulticlassNms + MulticlassNms, + Subgraph }; enum Algorithm { diff --git a/inference-engine/src/mkldnn_plugin/emitters/cpu_generator.cpp b/inference-engine/src/mkldnn_plugin/emitters/cpu_generator.cpp new file mode 100644 index 00000000000..a5117b9dcf9 --- /dev/null +++ b/inference-engine/src/mkldnn_plugin/emitters/cpu_generator.cpp @@ -0,0 +1,139 @@ +// Copyright (C) 2020-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/snippets_isa.hpp" + +#include +#include +#include + +#include "cpu_generator.hpp" +#include "jit_snippets_emitters.hpp" +#include "jit_eltwise_emitters.hpp" +#include "jit_mkldnn_emitters.hpp" +#include "jit_mkldnn_ext_emitters.hpp" + +using namespace std; +using namespace ngraph::snippets; + +#define CREATE_EMITTER(e_type) [this](const std::shared_ptr& n) \ + -> std::shared_ptr {return std::make_shared(h.get(), isa, n);}; + +class jit_snippet : public dnnl::impl::cpu::x64::jit_generator { +public: + DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_snippet) + + ~jit_snippet() = default; + + jit_snippet() : jit_generator() { + } + + void generate() override { + } +}; + +MKLDNNPlugin::CPUTargetMachine::CPUTargetMachine(dnnl::impl::cpu::x64::cpu_isa_t host_isa) + : TargetMachine(), h(new jit_snippet()), isa(host_isa) { + // data movement + jitters[ngraph::opset1::Parameter::get_type_info_static()] = CREATE_EMITTER(NopEmitter); + jitters[ngraph::snippets::op::BlockedParameter::get_type_info_static()] = CREATE_EMITTER(NopEmitter); + jitters[ngraph::opset1::Result::get_type_info_static()] = CREATE_EMITTER(NopEmitter); + // jitters[ngraph::opset1::Constant::get_type_info_static()] = CREATE_EMITTER(); // Not supported + + jitters[ngraph::snippets::op::Load::get_type_info_static()] = CREATE_EMITTER(LoadEmitter); + jitters[ngraph::snippets::op::VectorLoad::get_type_info_static()] = CREATE_EMITTER(LoadEmitter); + jitters[ngraph::snippets::op::ScalarLoad::get_type_info_static()] = CREATE_EMITTER(ScalarLoadEmitter); + jitters[ngraph::snippets::op::BroadcastLoad::get_type_info_static()] = CREATE_EMITTER(BroadcastLoadEmitter); + + jitters[ngraph::snippets::op::Store::get_type_info_static()] = CREATE_EMITTER(StoreEmitter); + jitters[ngraph::snippets::op::VectorStore::get_type_info_static()] = CREATE_EMITTER(StoreEmitter); + jitters[ngraph::snippets::op::ScalarStore::get_type_info_static()] = CREATE_EMITTER(ScalarStoreEmitter); + + jitters[ngraph::snippets::op::Scalar::get_type_info_static()] = CREATE_EMITTER(ScalarEmitter); + jitters[ngraph::snippets::op::BroadcastMove::get_type_info_static()] = CREATE_EMITTER(FakeBroadcastEmitter); + // jitters[ngraph::snippets::op::Nop::get_type_info_static()] = CREATE_EMITTER(NopEmitter); // Not supported + // jitters[ngraph::opset1::Broadcast::get_type_info_static()] = CREATE_EMITTER(); // Not supported + + // jitters[ngraph::opset1::Convert::get_type_info_static()] = CREATE_EMITTER(); // Not supported + // jitters[ngraph::opset1::FakeQuantize::get_type_info_static()] = CREATE_EMITTER(); // not supported + + // binary + jitters[ngraph::opset1::Add::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_add_emitter); + jitters[ngraph::opset1::Divide::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_divide_emitter); + jitters[ngraph::opset1::Equal::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_equal_emitter); + jitters[ngraph::opset1::FloorMod::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_floor_mod_emitter); + jitters[ngraph::opset1::Greater::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_greater_emitter); + jitters[ngraph::opset1::GreaterEqual::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_greater_equal_emitter); + jitters[ngraph::opset1::Less::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_less_emitter); + jitters[ngraph::opset1::LessEqual::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_less_equal_emitter); + jitters[ngraph::opset1::LogicalAnd::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_logical_and_emitter); + jitters[ngraph::opset1::LogicalOr::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_logical_or_emitter); + jitters[ngraph::opset1::LogicalXor::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_logical_xor_emitter); + jitters[ngraph::opset1::Maximum::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_maximum_emitter); + jitters[ngraph::opset1::Minimum::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_minimum_emitter); + jitters[ngraph::opset1::Mod::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_mod_emitter); + jitters[ngraph::opset1::Multiply::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_multiply_emitter); + jitters[ngraph::opset1::NotEqual::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_not_equal_emitter); + jitters[ngraph::snippets::op::PowerStatic::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_power_static_emitter); + jitters[ngraph::opset1::Power::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_power_dynamic_emitter); + jitters[ngraph::opset1::PRelu::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_prelu_emitter); + jitters[ngraph::opset1::SquaredDifference::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_squared_difference_emitter); + jitters[ngraph::opset1::Subtract::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_subtract_emitter); + jitters[ngraph::opset1::Xor::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_logical_xor_emitter); + + // unary + jitters[ngraph::opset1::Abs::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_abs_emitter); + // jitters[ngraph::opset1::Acos::get_type_info_static()] = CREATE_EMITTER(); // not supported + // jitters[ngraph::opset1::Asin::get_type_info_static()] = CREATE_EMITTER(); // not supported + // jitters[ngraph::opset1::Atan::get_type_info_static()] = CREATE_EMITTER(); // not supported + // jitters[ngraph::opset1::Ceiling::get_type_info_static()] = CREATE_EMITTER(); // not supported + jitters[ngraph::opset1::Clamp::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_clamp_emitter); + // jitters[ngraph::opset1::Cos::get_type_info_static()] = CREATE_EMITTER(); // not supported + // jitters[ngraph::opset1::Cosh::get_type_info_static()] = CREATE_EMITTER(); // not supported + jitters[ngraph::opset1::Elu::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_elu_emitter); + jitters[ngraph::opset1::Erf::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_erf_emitter); + jitters[ngraph::opset1::Exp::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_exp_emitter); + // jitters[ngraph::opset1::Floor::get_type_info_static()] = CREATE_EMITTER(); // not supported + // jitters[ngraph::opset1::Log::get_type_info_static()] = CREATE_EMITTER(); // not supported + jitters[ngraph::opset1::LogicalNot::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_logical_not_emitter); + jitters[ngraph::opset1::Negative::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_negative_emitter); + jitters[ngraph::opset1::Relu::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_relu_emitter); + // jitters[ngraph::opset1::Sign::get_type_info_static()] = CREATE_EMITTER(); // not supported + jitters[ngraph::opset1::Sigmoid::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_sigmoid_emitter); + // jitters[ngraph::opset1::Sin::get_type_info_static()] = CREATE_EMITTER(); // not supported + // jitters[ngraph::opset1::Sinh::get_type_info_static()] = CREATE_EMITTER(); // not supported + jitters[ngraph::opset1::Sqrt::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_sqrt_emitter); + // jitters[ngraph::opset1::Tan::get_type_info_static()] = CREATE_EMITTER(); // not supported + jitters[ngraph::opset1::Tanh::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_tanh_emitter); + + jitters[ngraph::op::v4::HSwish::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_hswish_emitter); + // jitters[ngraph::opset1::HardSigmoid::get_type_info_static()] = CREATE_EMITTER(); // not supported + // jitters[ngraph::opset1::Selu::get_type_info_static()] = CREATE_EMITTER(); // not supported + jitters[ngraph::op::v0::Gelu::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_gelu_v0_emitter); + jitters[ngraph::op::v7::Gelu::get_type_info_static()] = CREATE_EMITTER(MKLDNNPlugin::jit_gelu_v7_emitter); + + jitters[ngraph::snippets::op::Kernel::get_type_info_static()] = CREATE_EMITTER(KernelEmitter); + jitters[ngraph::snippets::op::Tile::get_type_info_static()] = CREATE_EMITTER(TileEmitter); +} + +size_t MKLDNNPlugin::CPUTargetMachine::get_lanes() const { + switch (isa) { + case dnnl::impl::cpu::x64::avx2 : return dnnl::impl::cpu::x64::cpu_isa_traits::vlen / sizeof(float); + case dnnl::impl::cpu::x64::sse41 : return dnnl::impl::cpu::x64::cpu_isa_traits::vlen / sizeof(float); + case dnnl::impl::cpu::x64::avx512_common : return dnnl::impl::cpu::x64::cpu_isa_traits::vlen / sizeof(float); + default : IE_THROW() << "unknown isa " << isa; + } +} + +bool MKLDNNPlugin::CPUTargetMachine::is_supported() const { + return dnnl::impl::cpu::x64::mayiuse(isa); +} + +code MKLDNNPlugin::CPUTargetMachine::get_snippet() const { + h->create_kernel(); + return h->jit_ker(); +} + +MKLDNNPlugin::CPUGenerator::CPUGenerator(dnnl::impl::cpu::x64::cpu_isa_t isa_) : Generator(std::make_shared(isa_)) { +} diff --git a/inference-engine/src/mkldnn_plugin/emitters/cpu_generator.hpp b/inference-engine/src/mkldnn_plugin/emitters/cpu_generator.hpp new file mode 100644 index 00000000000..3b2b94cb204 --- /dev/null +++ b/inference-engine/src/mkldnn_plugin/emitters/cpu_generator.hpp @@ -0,0 +1,33 @@ +// Copyright (C) 2020-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "snippets/generator.hpp" + +namespace MKLDNNPlugin { + +class CPUTargetMachine : public ngraph::snippets::TargetMachine { +public: + CPUTargetMachine(dnnl::impl::cpu::x64::cpu_isa_t host_isa); + + bool is_supported() const override; + ngraph::snippets::code get_snippet() const override; + size_t get_lanes() const override; + +private: + std::unique_ptr h; + dnnl::impl::cpu::x64::cpu_isa_t isa; +}; + +class CPUGenerator : public ngraph::snippets::Generator { +public: + CPUGenerator(dnnl::impl::cpu::x64::cpu_isa_t isa); + ~CPUGenerator() = default; +}; + +} // namespace MKLDNNPlugin \ No newline at end of file diff --git a/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp b/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp index 03eb2fdca44..c801082c01f 100644 --- a/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp +++ b/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp @@ -1286,21 +1286,14 @@ size_t jit_logical_not_emitter::aux_vecs_count() const { /// POWER_STATIC /// jit_power_static_emitter::jit_power_static_emitter(jit_generator *host, cpu_isa_t host_isa, const std::shared_ptr& node, Precision exec_prc) : jit_emitter(host, host_isa, node, exec_prc) { - auto parent = node->input(1).get_source_output().get_node_shared_ptr(); - if (!std::dynamic_pointer_cast(parent)) { - throw ngraph::ngraph_error("unsupported non constant power"); + auto powerStaticNode = ov::as_type_ptr(node); + if (powerStaticNode == nullptr) { + IE_THROW() << "Can't cast to snippets::op::PowerStatic"; } - if (!(node->input(1).get_shape() == ngraph::Shape() || ngraph::shape_size(node->input(1).get_shape()) == 1)) { - throw ngraph::ngraph_error("unsupported non scalar power"); - } - power = ngraph::as_type_ptr(parent)->get_data_ptr()[0]; + power = powerStaticNode->get_power(); scale = 1.f; shift = 0.f; - push_arg_entry_of("power", cpu::x64::float2int(power), true); - push_arg_entry_of("scale", 0x3f800000, true); - push_arg_entry_of("shift", 0x00000000, true); - push_arg_entry_of("one", 0x3f800000, true); prepare_table(); } @@ -1611,6 +1604,11 @@ jit_erf_emitter::jit_erf_emitter(jit_generator *host, cpu_isa_t host_isa, const prepare_table(); } +jit_erf_emitter::jit_erf_emitter(jit_generator *host, cpu_isa_t host_isa, const std::shared_ptr& node, Precision exec_prc) +: jit_emitter(host, host_isa, node, exec_prc) { + prepare_table(); +} + size_t jit_erf_emitter::get_inputs_num() const { return 1; } void jit_erf_emitter::emit_impl( diff --git a/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.hpp b/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.hpp index 50ddb57bad6..e03d32de9c1 100644 --- a/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.hpp +++ b/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.hpp @@ -531,6 +531,9 @@ public: jit_erf_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const MKLDNNNode* node, InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32); + jit_erf_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32); + size_t get_inputs_num() const override; private: diff --git a/inference-engine/src/mkldnn_plugin/emitters/jit_emitter.cpp b/inference-engine/src/mkldnn_plugin/emitters/jit_emitter.cpp index 49e853a24c9..236f6e873f0 100644 --- a/inference-engine/src/mkldnn_plugin/emitters/jit_emitter.cpp +++ b/inference-engine/src/mkldnn_plugin/emitters/jit_emitter.cpp @@ -101,7 +101,8 @@ void jit_emitter::emitter_preamble(const std::vector &in_idxs, const std aux_vec_idxs.push_back(idx); preserved_vec_idxs.push_back(idx); } - assert(aux_vec_idxs.size() >= aux_vecs_count()); + if (aux_vec_idxs.size() < aux_vecs_count()) + IE_THROW() << "Failed to allocate required number of vector registers"; // Same logic but to allocate gprs for (auto idx : pool_gpr_idxs) @@ -124,7 +125,8 @@ void jit_emitter::emitter_preamble(const std::vector &in_idxs, const std aux_gpr_idxs.push_back(_idx); preserved_gpr_idxs.push_back(_idx); } - assert(aux_gpr_idxs.size() == aux_gprs_count()); + if (aux_gpr_idxs.size() < aux_gprs_count()) + IE_THROW() << "Failed to allocate required number of general-purpose registers"; if (!entry_map_.empty()) { // last aux_gpr_idx is for p_table, we can use aux_gpr_idxs from idx 0 for other purpose diff --git a/inference-engine/src/mkldnn_plugin/emitters/jit_emitter.hpp b/inference-engine/src/mkldnn_plugin/emitters/jit_emitter.hpp index 026936caa32..12f422b7d7f 100644 --- a/inference-engine/src/mkldnn_plugin/emitters/jit_emitter.hpp +++ b/inference-engine/src/mkldnn_plugin/emitters/jit_emitter.hpp @@ -7,6 +7,8 @@ #include #include +#include "snippets/snippets_isa.hpp" +#include "snippets/generator.hpp" #include "mkldnn_node.h" #include @@ -24,23 +26,23 @@ struct emitter_context { virtual ~emitter_context() = default; }; -class jit_emitter { +class jit_emitter : public ngraph::snippets::Emitter { public: jit_emitter(dnnl::impl::cpu::x64::jit_generator* host, dnnl::impl::cpu::x64::cpu_isa_t host_isa, const MKLDNNNode* node, InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32, emitter_in_out_map in_out_type = emitter_in_out_map::vec_to_vec) - : h(host), host_isa_(host_isa), exec_prc_(exec_prc), in_out_type_(in_out_type), l_table (new Xbyak::Label()) { + : Emitter(nullptr), h(host), host_isa_(host_isa), exec_prc_(exec_prc), in_out_type_(in_out_type), l_table (new Xbyak::Label()) { k_mask = Xbyak::Opmask(1); // FIXME: in general case we need preserve k_mask state as well } jit_emitter(dnnl::impl::cpu::x64::jit_generator* host, dnnl::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32, emitter_in_out_map in_out_type = emitter_in_out_map::vec_to_vec) - : h(host), host_isa_(host_isa), exec_prc_(exec_prc), in_out_type_(in_out_type), l_table (new Xbyak::Label()) { + : Emitter(n), h(host), host_isa_(host_isa), exec_prc_(exec_prc), in_out_type_(in_out_type), l_table (new Xbyak::Label()) { k_mask = Xbyak::Opmask(1); // FIXME: in general case we need preserve k_mask state as well } - virtual void emit_code(const std::vector &in_idxs, const std::vector &out_idxs, - const std::vector &pool_vec_idxs = {}, const std::vector &pool_gpr_idxs = {}) const; - virtual void emit_data() const; + void emit_code(const std::vector &in_idxs, const std::vector &out_idxs, + const std::vector &pool_vec_idxs = {}, const std::vector &pool_gpr_idxs = {}) const override; + void emit_data() const override; virtual void emit_code(const std::vector &in_idxs, const std::vector &out_idxs, const std::shared_ptr &emit_context, @@ -147,6 +149,8 @@ private: const auto scale = te.bcast ? get_vec_length() : sizeof(table_entry_val_t); return te.off + key_off_val_shift * scale; } + virtual void validate_arguments(const std::vector&, const std::vector&, + const std::vector&, const std::vector &) const {} }; } // namespace MKLDNNPlugin diff --git a/inference-engine/src/mkldnn_plugin/emitters/jit_mkldnn_ext_emitters.hpp b/inference-engine/src/mkldnn_plugin/emitters/jit_mkldnn_ext_emitters.hpp new file mode 100644 index 00000000000..15c2dc98563 --- /dev/null +++ b/inference-engine/src/mkldnn_plugin/emitters/jit_mkldnn_ext_emitters.hpp @@ -0,0 +1,144 @@ +// Copyright (C) 2020-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "jit_mkldnn_emitters.hpp" + +namespace MKLDNNPlugin { + +class jit_relu_emitter : public jit_mkldnn_emitter { +public: + jit_relu_emitter(dnnl::impl::cpu::x64::jit_generator *host, dnnl::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = mkldnn_eltwise_relu; + alpha = 0.f; + beta = 0.f; + + set_injector(); + } +}; + +class jit_sigmoid_emitter : public jit_mkldnn_emitter { +public: + jit_sigmoid_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = mkldnn_eltwise_logistic; + alpha = 0.f; + beta = 0.f; + + set_injector(); + } +}; + +class jit_tanh_emitter : public jit_mkldnn_emitter { +public: + jit_tanh_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = mkldnn_eltwise_tanh; + alpha = 0.f; + beta = 0.f; + + set_injector(); + } +}; + +class jit_elu_emitter : public jit_mkldnn_emitter { +public: + jit_elu_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = mkldnn_eltwise_elu; + alpha = ngraph::as_type_ptr(n)->get_alpha(); + beta = 0.f; + + set_injector(); + } +}; + +class jit_exp_emitter : public jit_mkldnn_emitter { +public: + jit_exp_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = mkldnn_eltwise_exp; + alpha = 0.f; + beta = 0.f; + + set_injector(); + } +}; + +class jit_abs_emitter : public jit_mkldnn_emitter { +public: + jit_abs_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = mkldnn_eltwise_abs; + alpha = 0.f; + beta = 0.f; + + set_injector(); + } +}; + +class jit_clamp_emitter : public jit_mkldnn_emitter { +public: + jit_clamp_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = dnnl_eltwise_clip; + auto op = ngraph::as_type_ptr(n); + alpha = op->get_min(); + beta = op->get_max(); + + set_injector(); + } +}; + +class jit_hswish_emitter : public jit_mkldnn_emitter { +public: + jit_hswish_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = dnnl_eltwise_hardswish; + alpha = 0.f; + beta = 0.f; + + set_injector(); + } +}; +class jit_gelu_v0_emitter : public jit_mkldnn_emitter { +public: + jit_gelu_v0_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + kind = dnnl_eltwise_gelu_erf; + + set_injector(); + } +}; + +class jit_gelu_v7_emitter : public jit_mkldnn_emitter { +public: + jit_gelu_v7_emitter(mkldnn::impl::cpu::x64::jit_generator *host, mkldnn::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32) + : jit_mkldnn_emitter(host, host_isa, n, exec_prc) { + auto gelu = getNgraphOpAs(n); + ngraph::op::GeluApproximationMode approximationMode = gelu->get_approximation_mode(); + if (approximationMode == ngraph::op::GeluApproximationMode::ERF) + kind = dnnl_eltwise_gelu_erf; + else if (approximationMode == ngraph::op::GeluApproximationMode::TANH) + kind = dnnl_eltwise_gelu_tanh; + else + IE_THROW(NotImplemented) << "Subgraph node doesn't support ngraph operation Gelu with approximation mode: " << approximationMode; + + set_injector(); + } +}; + +} // namespace MKLDNNPlugin \ No newline at end of file diff --git a/inference-engine/src/mkldnn_plugin/emitters/jit_snippets_emitters.hpp b/inference-engine/src/mkldnn_plugin/emitters/jit_snippets_emitters.hpp new file mode 100644 index 00000000000..e2f131b9baf --- /dev/null +++ b/inference-engine/src/mkldnn_plugin/emitters/jit_snippets_emitters.hpp @@ -0,0 +1,641 @@ +// Copyright (C) 2020-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "jit_emitter.hpp" +using namespace Xbyak; +namespace MKLDNNPlugin { + +#define SNIPPETS_MAX_SNIPPETS_DIMS 7 +#define SNIPPETS_MAX_HARNESS_DIMS 5 +#define SNIPPETS_MAX_TILE_RANK 2 +#define GET_OFF(field) offsetof(jit_snippets_call_args, field) +struct jit_snippets_call_args { + const void *src_ptrs[SNIPPETS_MAX_SNIPPETS_DIMS] = {}; + void *dst_ptrs[SNIPPETS_MAX_SNIPPETS_DIMS] = {}; +}; + +struct jit_snippets_compile_args { + int64_t scheduler_dims[SNIPPETS_MAX_TILE_RANK] = {}; + int64_t scheduler_offsets[SNIPPETS_MAX_SNIPPETS_DIMS] = {}; + int64_t data_offsets[SNIPPETS_MAX_SNIPPETS_DIMS * SNIPPETS_MAX_HARNESS_DIMS] = {}; + std::vector output_dims = {}; +}; +/// +/// \brief Kernel is the only entry point to Codogen Jit compilation. Kernel calculates appropriate data offsets, +/// and invokes enclosed outer Tiles. Only 2d Tiles are currently supported, so the emitters should +/// be organized in the following way: +/// KernelEmitter { /* entry point */ +/// TileEmitter { /* outer tile */ +/// TileEmitter { /* inner vector tile */ +/// ... /* All the necessary Load/Strore/elementwise emitters */ +/// } +/// TileEmitter { /* inner scalar tile for tail processing */ +/// ... /* All the necessary Load/Strore/elementwise emitters */ +/// } +/// } +/// } +/// Note that Kernel params are passed directly to the emit_code(). The vector of inputs should contain 2 arguments, the +/// output vector should be empty. Input parameters +/// +/// \param in[0] The number of the node inputs +/// \param in[1] The number of the node outputs +/// +// Todo: Scheduler dims and offsets are currently calculated in MKLDNN Subgraph node and passed to the KernelEmitter. +// However, it seems more natural to calculate all the offsets right in the Kernel op, because the calculation is +// not device-specific. It is based only on input/output dims (which we already know) and harness num dims +// (which we should pass from the plugin). It seems also better to wrap the enclosed emitters in tiles in the Kernel op +// and avoid creating empty tiles. +class KernelEmitter : public jit_emitter { +public: + KernelEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, + const std::shared_ptr& n) + : jit_emitter(h, isa, n) { + const auto kernel = ov::as_type_ptr(n); + if (!kernel) + IE_THROW() << "KernelEmitter invoked with invalid op argument"; + if (!kernel->compile_params) + IE_THROW() << "KernelEmitter invoked without compile_params"; + code = kernel->region; + jcp = *reinterpret_cast(kernel->compile_params); + } + + size_t get_inputs_num() const override {return 0;} + + void emit_code(const std::vector &in, const std::vector &out, + const std::vector &pool = {}, const std::vector &gpr = {}) const override { + validate_arguments(in, out, pool, gpr); + emit_impl(in, out, pool, gpr, nullptr); + } + +private: + void validate_arguments(const std::vector &in, const std::vector &out, + const std::vector &pool = {}, const std::vector &gpr = {}) const override { + if (in.size() != 2) + IE_THROW() << "KernelEmitter got invalid number of inputs. Expected 2, got " << in.size(); + if (out.size() != 0) + IE_THROW() << "KernelEmitter got unexpected output arguments."; + const size_t num_params = in[0] + in[1]; + if (num_params > SNIPPETS_MAX_SNIPPETS_DIMS) + IE_THROW() << "KernelEmitter supports only up to " << SNIPPETS_MAX_SNIPPETS_DIMS << + " parameters, got " << num_params; + const int64_t harness_num_dims = jcp.output_dims.size() - 1; + if (harness_num_dims > SNIPPETS_MAX_HARNESS_DIMS) + IE_THROW() << "KernelEmitter supports harness with up to " << SNIPPETS_MAX_HARNESS_DIMS << + " dims, got " << harness_num_dims; + } + + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + const size_t num_inputs = in[0]; + const size_t num_outputs = in[1]; + const size_t num_params = num_inputs + num_outputs; + int reg64_tmp_start { 8 }; // R8, R9, R10, R11, R12, R13, R14, R15 inputs+outputs+1 + const int64_t harness_num_dims = jcp.output_dims.size() - 1; + + Reg64 reg_indexes { dnnl::impl::cpu::x64::abi_param1 }; + Reg64 reg_const_params { dnnl::impl::cpu::x64::abi_param2 }; + Xbyak::Reg64 reg_tmp_64 { dnnl::impl::cpu::x64::abi_not_param1}; + + h->preamble(); + + std::vector regs(num_params); + auto init_ptrs_with_offsets = [&](Reg64 pointer, const int64_t *offsets) { + for (int j = 0; j < harness_num_dims; j++) { + if (jcp.output_dims[j] != 1 && offsets[j] != 0) { + h->mov(reg_tmp_64, offsets[j]); + h->imul(reg_tmp_64, h->ptr[reg_indexes + j * sizeof(size_t)]); + h->add(pointer, reg_tmp_64); + } + } + }; + for (auto i = 0; i < num_params; i++) { + regs[i] = Reg64(reg64_tmp_start + i); + if (i < num_inputs) + h->mov(regs[i], h->ptr[reg_const_params + GET_OFF(src_ptrs) + i * sizeof(void*)]); + else + h->mov(regs[i], h->ptr[reg_const_params + GET_OFF(dst_ptrs) + (i - num_inputs) * sizeof(void*)]); + init_ptrs_with_offsets(regs[i], &jcp.data_offsets[i * harness_num_dims]); + } + + for (auto& c : code) { + c.first->emit_code(c.second.first, c.second.second, pool, gpr); + } + + h->postamble(); + } + + jit_snippets_compile_args jcp; + std::vector, ngraph::snippets::RegInfo>> code; +}; +/// +/// \brief Tile is designed to organize loop over the input and output data. It is essentially a for(...) loop: +/// it calculates the total number of iterations, performs operations specified by enclosed emitters, advances iteration counters +/// and breaks when necessary. +/// +/// \param in[0] The number of input entities (or scheduler counts) processed during one iteration of the tile. +/// It is expected to be 1 for outer or scalar tiles and vlen for vector tiles. +/// \param in[1] Increment of the previous Tile in current dimension. Must be 0 if this is the first Tile. +/// So previous_inc is zero for outer and vector tiles (the are the first in dim) and vlen for scalar tiles (they usually go after vector Tiles). +/// \param in[2] sum number inputs and number of outputs of the node. +/// \param in[3] dimension of the tile. Note that only 2d Tile are currently supported, so dim is 0 for outer tiles, 1 for inner tiles. +/// +// Todo: Inner and outer tiles have different semantics. For example, outer tile always has the increment == 1, and it can contain only +// tile emitters (one outer or two inner). So it seems better to create different classes for inner and outer tiles. +// Todo: Currently data pointers incremented after each read/write in Load/Store emitters, so we have to decrement them here +// if the same data needs to be read twice. Better to move all the pointer increments to TileEmitter and avoid the increments if necessary. +class TileEmitter : public jit_emitter { +public: + TileEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, + const std::shared_ptr& n) + : jit_emitter(h, isa, n) { + const auto tile = ov::as_type_ptr(n); + if (!tile) + IE_THROW() << "TileEmitter invoked with invalid op argument"; + if (!tile->compile_params) + IE_THROW() << "TileEmitter invoked without compile_params"; + code = tile->region; + jcp = *reinterpret_cast(tile->compile_params); + } + + size_t get_inputs_num() const override {return 0;} + + void emit_code(const std::vector &in, const std::vector &out, + const std::vector &pool = {}, const std::vector &gpr = {}) const override { + validate_arguments(in, out, pool, gpr); + emit_impl(in, out, pool, gpr, nullptr); + } + +private: + void validate_arguments(const std::vector &in, const std::vector &out, + const std::vector &pool = {}, const std::vector &gpr = {}) const override { + if (in.size() != 4) + IE_THROW() << "TileEmitter got invalid number of inputs. Expected 4, got " << in.size(); + if (out.size() != 0) + IE_THROW() << "TileEmitter got unexpected output arguments."; + const size_t num_params = in[2]; + if (num_params > SNIPPETS_MAX_SNIPPETS_DIMS) + IE_THROW() << "TileEmitter supports only up to " << SNIPPETS_MAX_SNIPPETS_DIMS << + " parameters, got " << num_params; + const size_t dim = in[3]; + if (dim >= SNIPPETS_MAX_TILE_RANK) + IE_THROW() << "TileEmitter supports tile ranks up to " << SNIPPETS_MAX_TILE_RANK << + " got " << dim; + } + + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + const size_t inc = in[0]; + const size_t previous_inc = in[1]; // increment of a previous tile in the same dim (0 if the first tile in the dim) + const size_t num_params = in[2]; + const size_t dim = in[3]; // tile dimension: 0 - outer, 1 - inner + const int reg64_tmp_start { 8 }; // R8, R9, R10, R11, R12, R13, R14, R15 inputs+outputs+1 + Reg64 amount = Reg64(reg64_tmp_start + num_params); // amount + std::array for_body; + + // If R15 is not used, reserve it for use in scalar to avoid redundant push-pop's. + // todo: Do we need explicitly check that code contains ScalarEmitter? + std::vector local_gpr = reg64_tmp_start + num_params < 15 ? std::vector{15} : std::vector{}; + std::vector regs(num_params); + for (auto i = 0; dim == 0 && i < num_params; i++) + regs[i] = Reg64(reg64_tmp_start + i); + // Loop processing could be simplified in some cases + if (inc > jcp.scheduler_dims[dim]) { + return; + } else if (inc == jcp.scheduler_dims[dim]) { + for (auto& c : code) { + c.first->emit_code(c.second.first, c.second.second, pool, local_gpr); + } + } else { + // The previous tile has done nothing, all the work is ours + if (previous_inc == 0 || previous_inc > jcp.scheduler_dims[dim]) { + h->mov(amount, jcp.scheduler_dims[dim]); + // The previous tile has done all the work + } else if (jcp.scheduler_dims[dim] % previous_inc == 0) { + return; + }// else: the previous tile has already set a proper work amount + h->cmp(amount, inc); + h->jl(for_body[0], CodeGenerator::T_NEAR); + + h->L(for_body[1]); + { + h->push(amount); + for (auto& c : code) { + c.first->emit_code(c.second.first, c.second.second, pool, local_gpr); + } + h->pop(amount); + // Todo: Load and Store emitters are currently implemented so they ALWAYS increment appropriate pointers + // after reading/writing. This might be a problem if we need to read the same data multiple times (broadcasting shapes). + // To overcome this limitation, we add appropriate negative offsets if necessary. + for (auto i = 0; dim == 0 && i < num_params; i++) { + if (jcp.scheduler_offsets[i] != 0) { + h->add(regs[i], jcp.scheduler_offsets[i]); + } + } + h->sub(amount, inc); + h->cmp(amount, inc); + h->jge(for_body[1], CodeGenerator::T_NEAR); + } + + h->L(for_body[0]); + } + } + + // A = <42, 17> + // B = < 1, 17> + // for (auto k = 0; k < dom_0; k++) { // 42 + // for (auto n = 0; n < dom_1; n++) { // 17 + // auto a = *ptr0; ptr0 += vlan; // vector/scalar load + // auto b = *ptr1; ptr1 += vlan; // vector/scalar load + // } + // ptr0 -= 0*dom_1; + // ptr1 -= 1*dom_1; + // } + + // broadcast by MVD is extra case + // A = <42, 17> + // B = <42, 1> + // for (auto k = 0; k < dom_0; k++) { // 42 + // for (auto n = 0; n < dom_1; n++) { // 17 + // auto a = *ptr0; ptr0 += vlan; // vector/scalar load + // auto b = *ptr1; // broadcast load + // } + // ptr0 -= 0*dom_1; + // ptr1 += sizeof(ptr1[0]); //ptr1 -= -sizeof(ptr1[0]); + // } + + // A = <42, 17, 31> + // B = < 1, 17, 31> + // for (auto k = 0; k < dom_0; k++) { // 42 + // for (auto n = 0; n < dom_1; n++) { // 17 + // for (auto m = 0; m < dom_2; m++) { // 31 + // auto a = *ptr0; ptr0 += vlan; // vector/scalar load + // auto b = *ptr1; ptr1 += vlan; // vector/scalar load + // } + // } + // ptr0 -= 0*dom_1*dom2; + // ptr1 -= 1*dom_1*dom2; + // } + jit_snippets_compile_args jcp; + std::vector, ngraph::snippets::RegInfo>> code; +}; + +class NopEmitter : public jit_emitter { +public: + NopEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : jit_emitter(h, isa, n) { + } + + size_t get_inputs_num() const override {return 0;} + +private: + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + } +}; + +class FakeBroadcastEmitter : public jit_emitter { +public: + FakeBroadcastEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : jit_emitter(h, isa, n) { + if (n->get_input_shape(0).empty()) + use_broadcast = true; + else if (*n->get_input_shape(0).rbegin() != *n->get_output_shape(0).rbegin()) + use_broadcast = true; + else + use_broadcast = false; + } + size_t get_inputs_num() const override {return 1;} + +private: + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + if (host_isa_ == dnnl::impl::cpu::x64::sse41) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx2) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx512_common) { + emit_isa(in, out); + } else { + IE_THROW() << host_isa_; + assert(!"unsupported isa"); + } + } + + template + void emit_isa(const std::vector &in, const std::vector &out) const { + using Vmm = typename dnnl::impl::utils::conditional3::type; + Vmm vmm_src0 = Vmm(in[0]); + Vmm vmm_dst = Vmm(out[0]); + + if (use_broadcast) { + h->uni_vbroadcastss(vmm_dst, Xmm(in[0])); + } else { + h->uni_vmovups(vmm_dst, vmm_src0); + } + } + +private: + bool use_broadcast; +}; + +class ScalarEmitter : public jit_emitter { +public: + ScalarEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : jit_emitter(h, isa, n) { + auto out_pshape = n->output(0).get_tensor().get_partial_shape(); + if (out_pshape.is_dynamic()) + IE_THROW() << "ScalarEmitter supports only static input shapes"; + if ( out_pshape.get_shape() != ov::Shape() && ov::shape_size(out_pshape.get_shape()) != 1) + IE_THROW() << "ScalarEmitter got invalid shape"; + value = mkldnn::impl::cpu::x64::float2int(ov::as_type_ptr(n)->cast_vector()[0]); + + push_arg_entry_of("scalar", value, true); + prepare_table(); + } + + size_t get_inputs_num() const override {return 0;} + +protected: + size_t aux_gprs_count() const override {return 1;} + +private: + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + if (host_isa_ == dnnl::impl::cpu::x64::sse41) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx2) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx512_common) { + emit_isa(in, out); + } else { + IE_THROW() << host_isa_; + assert(!"unsupported isa"); + } + } + + template + void emit_isa(const std::vector &in, const std::vector &out) const { + using Vmm = typename dnnl::impl::utils::conditional3::type; + Vmm vmm_dst = Vmm(out[0]); + h->uni_vbroadcastss(vmm_dst, table_val("scalar")); + } + +private: + int32_t value; +}; + +/// +/// Memory emitters: +/// +/// *Note*: post increment is embedded into Load/Store operation which means that +/// it's illigal to load/store to the same address multiple times +/// Typical application can be if Load and BroadcastLoad are performed from the same pointer. +/// If Load goes before BroadcastLoad topologicaly the resilt will be incorrect +/// For scalar loads we can use different tiles. Tiling indeed can be arbitrary and post increment should be somehow coded into ISA. +/// Blocked parameter to tell if input is actually blocked. Broadcast means broadcast by W in other cases no need to substitute load. +class MemoryEmitter : public jit_emitter { +public: + MemoryEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : jit_emitter(h, isa, n), ea(getEA(n)) { + } + + size_t get_inputs_num() const override {return 1;} + +protected: + static auto getEA(const std::shared_ptr& n) -> size_t { + auto& rt = n->get_rt_info(); + size_t ea = 0; + auto it = rt.find("effectiveAddress"); + if (it != rt.end()) { + ea = it->second.as(); + } else { + throw ov::Exception("effective address for Load generation cannot be determined"); + } + return ea; + } + + size_t ea; +}; + +class StoreEmitter : public MemoryEmitter { +public: + StoreEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : MemoryEmitter(h, isa, n) { + } + + size_t get_inputs_num() const override {return 1;} + +private: + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + if (host_isa_ == dnnl::impl::cpu::x64::sse41) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx2) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx512_common) { + emit_isa(in, out); + } else { + IE_THROW() << host_isa_; + assert(!"unsupported isa"); + } + } + + template + void emit_isa(const std::vector &in, const std::vector &out) const { + using Vmm = typename dnnl::impl::utils::conditional3::type; + Reg64 out_reg(ea); + Vmm vmm_src0 = Vmm(in[0]); + h->uni_vmovups(h->ptr[out_reg], vmm_src0); + h->add(out_reg, mkldnn::impl::cpu::x64::cpu_isa_traits::vlen); + } +}; + +class ScalarStoreEmitter : public MemoryEmitter { +public: + ScalarStoreEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : MemoryEmitter(h, isa, n) { + } + + size_t get_inputs_num() const override {return 1;} + +private: + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + if (host_isa_ == dnnl::impl::cpu::x64::sse41) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx2) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx512_common) { + emit_isa(in, out); + } else { + IE_THROW() << host_isa_; + assert(!"unsupported isa"); + } + } + + template + void emit_isa(const std::vector &in, const std::vector &out) const { + using Vmm = typename dnnl::impl::utils::conditional3::type; + Reg64 out_reg(ea); + Xmm vmm_src0 = Xmm(in[0]); + h->uni_vmovss(h->ptr[out_reg], vmm_src0); + h->add(out_reg, sizeof(float)); + } +}; + +class LoadEmitter : public MemoryEmitter { +public: + LoadEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : MemoryEmitter(h, isa, n), shouldPostIncrement(*n->get_input_shape(0).rbegin() != 1) { + } + + size_t get_inputs_num() const override {return 0;} + +private: + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + if (host_isa_ == dnnl::impl::cpu::x64::sse41) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx2) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx512_common) { + emit_isa(in, out); + } else { + IE_THROW() << host_isa_; + assert(!"unsupported isa"); + } + } + + template + void emit_isa(const std::vector &in, const std::vector &out) const { + using Vmm = typename dnnl::impl::utils::conditional3::type; + Reg64 in_reg(ea); + Vmm vmm_src0 = Vmm(out[0]); + h->uni_vmovups(vmm_src0, h->ptr[in_reg]); + + if (shouldPostIncrement) { + h->add(in_reg, mkldnn::impl::cpu::x64::cpu_isa_traits::vlen); + } + } + +private: + bool shouldPostIncrement; +}; + +class BroadcastLoadEmitter : public MemoryEmitter { +public: + BroadcastLoadEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : MemoryEmitter(h, isa, n) { + } + size_t get_inputs_num() const override {return 0;} + +private: + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + if (host_isa_ == dnnl::impl::cpu::x64::sse41) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx2) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx512_common) { + emit_isa(in, out); + } else { + IE_THROW() << host_isa_; + assert(!"unsupported isa"); + } + } + + template + void emit_isa(const std::vector &in, const std::vector &out) const { + using Vmm = typename dnnl::impl::utils::conditional3::type; + Reg64 in_reg(ea); + Vmm vmm_src0 = Vmm(out[0]); + + // In doesn't really matter if we broadcast or `movss` for vector tails so keep only one version for `BroadcastLoad`, + // key point here is not to add post-increment, it might be fixed by some other approach in future + h->uni_vbroadcastss(vmm_src0, h->ptr[in_reg]); + } +}; + +class ScalarLoadEmitter : public MemoryEmitter { +public: + ScalarLoadEmitter(mkldnn::impl::cpu::x64::jit_generator* h, mkldnn::impl::cpu::x64::cpu_isa_t isa, const std::shared_ptr& n) + : MemoryEmitter(h, isa, n), shouldPostIncrement(*n->get_input_shape(0).rbegin() != 1) { + } + size_t get_inputs_num() const override {return 0;} + +private: + void emit_impl(const std::vector& in, + const std::vector& out, + const std::vector& pool, + const std::vector& gpr, + const MKLDNNPlugin::emitter_context *emit_context) const override { + if (host_isa_ == dnnl::impl::cpu::x64::sse41) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx2) { + emit_isa(in, out); + } else if (host_isa_ == dnnl::impl::cpu::x64::avx512_common) { + emit_isa(in, out); + } else { + IE_THROW() << host_isa_; + assert(!"unsupported isa"); + } + } + + template + void emit_isa(const std::vector &in, const std::vector &out) const { + using Vmm = typename dnnl::impl::utils::conditional3::type; + Reg64 in_reg(ea); + Xmm vmm_src0 = Xmm(out[0]); + h->uni_vmovss(vmm_src0, h->ptr[in_reg]); + + // Doesn't work if the same pointer comes with multiple load operations + if (shouldPostIncrement) { + h->add(in_reg, sizeof(float)); + } + } + +private: + bool shouldPostIncrement; +}; + +} // namespace MKLDNNPlugin \ No newline at end of file diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_nodes_factory.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_nodes_factory.cpp index f3ff2beba9a..85df2cbbad1 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_nodes_factory.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_nodes_factory.cpp @@ -81,6 +81,7 @@ #include "nodes/mkldnn_if_node.h" #include "nodes/mkldnn_ctc_greedy_decoder_node.h" #include "nodes/mkldnn_non_zero.h" +#include "nodes/subgraph.h" #define MKLDNN_NODE(__prim, __type) \ registerNodeIfRequired(MKLDNNPlugin, __prim, __type, MKLDNNNodeImpl<__prim>) @@ -170,4 +171,5 @@ MKLDNNPlugin::MKLDNNNode::NodesFactory::NodesFactory() MKLDNN_NODE(MKLDNNStridedSliceNode, StridedSlice); MKLDNN_NODE(MKLDNNGRNNode, GRN); MKLDNN_NODE(MKLDNNNonZeroNode, NonZero); + MKLDNN_NODE(MKLDNNSnippetNode, Subgraph); } diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp index 6f8af6141d9..7d948150c1e 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp @@ -80,6 +80,8 @@ #include #include #include +#include +#include "ngraph_transformations/snippets_mark_skipped.hpp" #include #include @@ -125,6 +127,7 @@ using namespace MKLDNNPlugin; using namespace InferenceEngine; + Engine::Engine() { _pluginName = "CPU"; extensionManager->AddExtension(std::make_shared()); @@ -136,7 +139,8 @@ Engine::~Engine() { ExecutorManager::getInstance()->clear("CPUCallbackExecutor"); } -static void TransformationUpToCPUSpecificOpSet(std::shared_ptr nGraphFunc, const bool _enableLPT) { +static void TransformationUpToCPUSpecificOpSet(std::shared_ptr nGraphFunc, const bool _enableLPT, + const bool _enableSnippets) { ngraph::pass::Manager manager; manager.set_per_pass_validation(false); manager.register_pass(); @@ -468,6 +472,7 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr ngraph::pass::Manager postLPTPassManager; postLPTPassManager.register_pass(); postLPTPassManager.register_pass(); + postLPTPassManager.register_pass(); postLPTPassManager.get_pass_config()->set_callback([](const_node_ptr &node) -> bool { std::string errMsg; @@ -478,6 +483,7 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr return node->get_rt_info().count("UNROLL_TI") == 0; }); + postLPTPassManager.register_pass(); postLPTPassManager.get_pass_config()->set_callback([](const std::shared_ptr& node) -> bool { if (node->get_input_size() >= 2) { @@ -486,12 +492,41 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr return false; }); + postLPTPassManager.register_pass(); postLPTPassManager.run_passes(nGraphFunc); + + if (!useLpt && _enableSnippets && with_cpu_x86_avx2()) { + ngraph::pass::Manager tokenization_manager; + tokenization_manager.register_pass(); + tokenization_manager.register_pass(); + tokenization_manager.register_pass(); + tokenization_manager.get_pass_config()->set_callback( + [](const std::shared_ptr& n) -> bool { + const auto& inputs = n->inputs(); + // todo: clarify whether we can evaluate snippets on const paths + const bool has_only_const_inputs = std::all_of(inputs.begin(), inputs.end(), + [](const ov::Input &in) { + return ov::is_type(in.get_source_output().get_node_shared_ptr()); + }); + // todo: clarify whether we can evaluate snippets on inputs with larger ranks + auto rank_is_too_large = [](const ov::descriptor::Tensor& t ) { + // callback is called has_supported_in_out(), so it's safe to assume that the shapes are static + return t.get_partial_shape().rank().get_length() > 6; + }; + const bool bad_input_rank = std::any_of(inputs.begin(), inputs.end(), + [&](const ov::Input& in) {return rank_is_too_large(in.get_tensor());}); + const auto& outputs = n->outputs(); + const bool bad_output_rank = std::any_of(outputs.begin(), outputs.end(), + [&](const ov::Output& out) {return rank_is_too_large(out.get_tensor());}); + return has_only_const_inputs || bad_input_rank || bad_output_rank; + }); + tokenization_manager.run_passes(nGraphFunc); + } } -static void Transformation(CNNNetwork& clonedNetwork, const bool _enableLPT) { +static void Transformation(CNNNetwork& clonedNetwork, const bool _enableLPT, const bool _enableSnippets) { auto nGraphFunc = clonedNetwork.getFunction(); - TransformationUpToCPUSpecificOpSet(nGraphFunc, _enableLPT); + TransformationUpToCPUSpecificOpSet(nGraphFunc, _enableLPT, _enableSnippets); ConvertToCPUSpecificOpset(nGraphFunc); } @@ -526,8 +561,18 @@ Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork &network, const std const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE); const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/ || Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled for the plugin */; + const auto& BF16Prop = config.find(InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16); + const bool enableBF16 = ((BF16Prop != config.end() && BF16Prop->second == PluginConfigParams::YES) + || engConfig.enforceBF16) && with_cpu_x86_avx512_core(); + const auto& modelCacheProp = config.find(InferenceEngine::PluginConfigParams::KEY_CACHE_DIR); + const bool enableModelCache = (modelCacheProp != config.end() && !modelCacheProp->second.empty()) + || !engConfig.cache_dir.empty(); + const auto& dynamicBatchProp = config.find(InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_ENABLED); + const bool enableDynamicBatch = (dynamicBatchProp != config.end() && dynamicBatchProp->second == PluginConfigParams::YES) + || engConfig.enableDynamicBatch; + const bool enableSnippets = !(enableModelCache || enableDynamicBatch || enableBF16); auto nGraphFunc = clonedNetwork.getFunction(); - TransformationUpToCPUSpecificOpSet(nGraphFunc, enableLPT); + TransformationUpToCPUSpecificOpSet(nGraphFunc, enableLPT, enableSnippets); // Here the OV perf modes are turned into specific settings (as we need the network for better params selection) const auto& mode = config.find(PluginConfigParams::KEY_PERFORMANCE_HINT); @@ -736,7 +781,8 @@ QueryNetworkResult Engine::QueryNetwork(const CNNNetwork& network, const std::ma const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE); const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/ || Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */; - Transformation(clonedNetwork, enableLPT); + const bool enableSnippets = !(conf.cache_dir.empty() || conf.enableDynamicBatch || (conf.enforceBF16 && with_cpu_x86_avx512_core())); + Transformation(clonedNetwork, enableLPT, enableSnippets); auto ops = clonedNetwork.getFunction()->get_ordered_ops(); std::unordered_set supported; std::unordered_set unsupported; diff --git a/inference-engine/src/mkldnn_plugin/ngraph_transformations/convert_to_cpu_specific_opset.hpp b/inference-engine/src/mkldnn_plugin/ngraph_transformations/convert_to_cpu_specific_opset.hpp index 121247c702c..9702ebf0d89 100644 --- a/inference-engine/src/mkldnn_plugin/ngraph_transformations/convert_to_cpu_specific_opset.hpp +++ b/inference-engine/src/mkldnn_plugin/ngraph_transformations/convert_to_cpu_specific_opset.hpp @@ -23,14 +23,12 @@ namespace MKLDNNPlugin { inline void ConvertToCPUSpecificOpset(std::shared_ptr &nGraphFunc) { ngraph::pass::Manager manager; - manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); - manager.register_pass(); manager.register_pass(); manager.register_pass(); if (!ngraph::op::util::has_op_with_type(nGraphFunc)) { diff --git a/inference-engine/src/mkldnn_plugin/ngraph_transformations/snippets_mark_skipped.cpp b/inference-engine/src/mkldnn_plugin/ngraph_transformations/snippets_mark_skipped.cpp new file mode 100644 index 00000000000..7a284e3a89f --- /dev/null +++ b/inference-engine/src/mkldnn_plugin/ngraph_transformations/snippets_mark_skipped.cpp @@ -0,0 +1,352 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "snippets_mark_skipped.hpp" +#include +#include +#include +#include + +NGRAPH_RTTI_DEFINITION(MKLDNNPlugin::SnippetsMarkSkipped, "SnippetsMarkSkipped", 0); + +using namespace ngraph; +namespace MKLDNNPlugin { +namespace { +NodeFusingType GetNodeFusingType(const std::shared_ptr &node) { + auto &rt = node->get_rt_info(); + const auto rinfo = rt.find("MayBeFusedInPlugin"); + if (rinfo == rt.end()) + return NodeFusingType::NotSet; + return rinfo->second.as(); +} +void SetNodeFusingType(const std::shared_ptr &node, NodeFusingType nodeType) { + auto &rt = node->get_rt_info(); + rt["MayBeFusedInPlugin"] = nodeType; +} +std::vector getContinuableChains(const std::shared_ptr &node) { + std::vector result; + for (const auto& input : node->inputs()) { + const auto parent = input.get_source_output().get_node_shared_ptr(); + const auto snt = GetNodeFusingType(parent); + if (snt > NodeFusingType::FusedTerminator) { + result.push_back(snt); + } + } + return result; +} +int getNumNonConstInputs(const std::shared_ptr &node) { + int num_non_const_inputs = 0; + for (const auto &parent_out : node->input_values()) { + const auto parent = parent_out.get_node_shared_ptr(); + if (ov::is_type(parent)) { + for (const auto &grandparent_out : parent->input_values()) { + const auto grandparent = grandparent_out.get_node_shared_ptr(); + if (!ngraph::op::is_constant(grandparent)) + num_non_const_inputs++; + } + } else if (!ngraph::op::is_constant(parent)) { + num_non_const_inputs++; + } + } + return num_non_const_inputs; +} +bool SupportsFusingWithConvolution_SumActivation(const std::shared_ptr &node) { + // todo: Do all PReLUs are fused? Not sure about round and softRelu + // EltwiseRoundHalfToEven, EltwiseRoundHalfAwayFromZero, EltwiseSoftRelu + return ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node); +} + +bool canBePerformedAsScaleShift(const std::shared_ptr &node, const size_t channelAxis) { + size_t fusingPort = 0; + size_t numNonConstInputs = 0; + ov::PartialShape dataShape; + for (size_t i = 0; i < node->get_input_size(); i++) { + const auto parent = node->get_input_node_shared_ptr(i); + if (!ngraph::op::is_constant(parent)) { + fusingPort = i; + dataShape = node->get_input_partial_shape(i); + // only one non-const parent is allowed + if (dataShape.is_dynamic() || ++numNonConstInputs != 1) + return false; + } else { + // every const parent must have exactly one child + const auto out = parent->outputs(); + const bool has_only_child = (out.size() == 1) && (out[0].get_target_inputs().size() == 1); + if (!has_only_child) + return false; + } + } + + const auto isBroadcastableToDataInput = [&]() { + for (size_t i = 0; i < node->get_input_size(); i++) { + if (i == fusingPort) + continue; + const ov::PartialShape weightShape = node->get_input_partial_shape(i); + if (weightShape.is_dynamic() || + !isPerTensorOrPerChannelBroadcastable(dataShape.get_shape(), weightShape.get_shape(), channelAxis, true)) + return false; + } + return true; + }; + + // Prelu and MulAdd are still ignored + // isConvertablePowerStatic() is ignored + return (ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node)) && + isBroadcastableToDataInput(); +} + +bool SupportsFusingWithConvolution_Simple(const std::shared_ptr &node, const size_t channelAxis = 1) { + return SupportsFusingWithConvolution_SumActivation(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + canBePerformedAsScaleShift(node, channelAxis); +} +// Convolution is a special case, since it supports peculiar fusings +bool isSuitableConvolutionParent(const std::shared_ptr &node) { + const bool is_suitable_node = ov::is_type(node) || + ov::is_type(node); + // has a single output, connected to a single child + const auto out = node->outputs(); + const bool has_only_child = (out.size() == 1) && (out[0].get_target_inputs().size() == 1); + return is_suitable_node && has_only_child; +} +bool isSuitableBinaryConvolutionParent(const std::shared_ptr &node) { + const bool is_suitable_node = ov::is_type(node); + // has a single output, connected to a single child + const auto out = node->outputs(); + const bool has_only_child = (out.size() == 1) && (out[0].get_target_inputs().size() == 1); + return is_suitable_node && has_only_child; +} +bool isSuitableMiscParent(const std::shared_ptr &node) { + const bool is_suitable_node = ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node); + // has a single output, connected to a single child + const auto out = node->outputs(); + const bool has_only_child = (out.size() == 1) && (out[0].get_target_inputs().size() == 1); + return is_suitable_node && has_only_child; +} +// Matmul is a special case, since it supports simple + bias fusings +bool isSuitableMatMulParent(const std::shared_ptr &node) { + const bool is_suitable_node = ov::is_type(node); + // has a single output, connected to a single child + const auto out = node->outputs(); + const bool has_only_child = (out.size() == 1) && (out[0].get_target_inputs().size() == 1); + return is_suitable_node && has_only_child; +} +bool isSuitablePoolChild(const std::shared_ptr &node) { + const bool is_suitable_node = ov::is_type(node); + // has a single output, connected to a single child + const auto out = node->outputs(); + const bool has_only_child = (out.size() == 1) && (out[0].get_target_inputs().size() == 1); + return is_suitable_node && has_only_child; +} +bool isSuitableChildForFusingSimple(const std::shared_ptr &node, const size_t channelAxis = 1) { + // Note: Fusing child is allowed to have several users, but that must be the end of the chain + return SupportsFusingWithConvolution_Simple(node, channelAxis) && getNumNonConstInputs(node) == 1; +} +bool isSuitableChildForFusingMatMul(const std::shared_ptr &node, NodeFusingType &updatedChainType) { + int num_non_const_inputs = 0; + bool can_be_converted_to_FC = false; + ov::Shape bias_shape; + ov::Shape matmul_shape; + for (const auto &parent_out : node->input_values()) { + const auto parent = parent_out.get_node_shared_ptr(); + if (ngraph::op::is_constant(parent)) { + bias_shape = parent_out.get_shape(); + num_non_const_inputs++; + } else { + const auto pshape = parent_out.get_partial_shape(); + if (pshape.is_dynamic() || pshape.get_shape().empty()) + return false; + matmul_shape = pshape.get_shape(); + const auto& grandparents = parent->input_values(); + // first check that weights are constant and both activations and weights have static shape + if (grandparents.size() == 2 && + grandparents[0].get_partial_shape().is_static() && + grandparents[1].get_partial_shape().is_static() && + ov::is_type(grandparents[1].get_node_shared_ptr())) { + auto rank_a = grandparents[0].get_partial_shape().rank().get_length(); + auto rank_w = grandparents[1].get_partial_shape().rank().get_length(); + if (rank_a != 1 && rank_w != 1 && rank_a <= 3 && rank_w <= 3) + can_be_converted_to_FC = true; + } + } + } + if (num_non_const_inputs != 1) + return false; + + // FuseMatMulAndSimpleOperation or FuseFullyConnectedAndSimpleOperation + // Invoke SupportsFusingWithConvolution_Simple directly instead of isSuitableChildForFusingSimple to + // eliminate getNumNonConstInputs() check + size_t fusingAxis; + if (can_be_converted_to_FC) + fusingAxis = matmul_shape.size() == 3 ? 2 : 1; + else + fusingAxis = matmul_shape.size() - 1; + if (SupportsFusingWithConvolution_Simple(node, fusingAxis)) { + updatedChainType = NodeFusingType::FusedWithMisc; + return true; + } + // FullyConnectedBiasFusion + if (!(can_be_converted_to_FC && ov::is_type(node) && + bias_shape.back() == matmul_shape.back() && + bias_shape.back() == shape_size(bias_shape))) { + return false; + } + // Fusing chain must be interrupted after the node, since reshape will be inserted + if (bias_shape.size() >= 2) + updatedChainType = NodeFusingType::FusedTerminator; + return true; +} +bool isSuitableParentForFusingSumActivation(const std::shared_ptr &node) { + if (!ov::is_type(node)) + return false; + auto isFusedBiasNode = [](std::shared_ptr n){ + if (!(ov::is_type(n) && + GetNodeFusingType(n) == NodeFusingType::FusedWithConvolution)) + return false; + const auto conv = n->get_input_source_output(0); + const auto bias = n->get_input_source_output(1); + if (!(ngraph::op::is_constant(bias.get_node_shared_ptr()) && isSuitableConvolutionParent(conv.get_node_shared_ptr()))) + return false; + const auto conv_shape = conv.get_partial_shape(); + const auto bias_shape = bias.get_partial_shape(); + if (bias_shape.is_dynamic() || conv_shape.is_dynamic() || bias_shape.size() > conv_shape.size()) + return false; + auto getNormalizedDims = [](const ov::Shape &dims, size_t ndims) -> std::vector{ + std::vector normalizedDims = dims; + for (size_t i = 0; i < (ndims - dims.size()); i++) { + normalizedDims.insert(normalizedDims.begin(), 1); + } + return normalizedDims; + }; + const auto bias_norm_dims = getNormalizedDims(bias_shape.get_shape(), conv_shape.size()); + if (bias_norm_dims.size() < 2 || bias_norm_dims[0] != 1 || conv_shape[1] != bias_norm_dims[1]) + return false; + for (size_t i = 2; i < bias_norm_dims.size(); i++) { + if (bias_norm_dims[i] != 1) + return false; + } + return true; + }; + int num_conv_parents = 0; + for (size_t i = 0; i < node->get_input_size(); i++) { + const auto n = node->get_input_node_shared_ptr(i); + //BinaryConvolution allows other ops to be fused before the Add, while Convolution doesn't + num_conv_parents += (isSuitableConvolutionParent(n) || isFusedBiasNode(n) || + GetNodeFusingType(n) == NodeFusingType::FusedWithBinaryConvolution); + } + return getNumNonConstInputs(node) == 2 && num_conv_parents >=1; +} +bool isSuitableChildForFusingSumActivation(const std::shared_ptr &node) { + return SupportsFusingWithConvolution_SumActivation(node); +} +// Continue fusing chain of the passed type if the node has one child +// Otherwise mark node as FusedTerminator (Fused, but fusing chain is interrupted) +void PropagateIfHasOnlyChild(const std::shared_ptr &node, NodeFusingType nodeType) { + const auto out = node->outputs(); + const bool has_only_child = out.size() == 1 && out[0].get_target_inputs().size() == 1; + SetNodeFusingType(node, has_only_child ? nodeType : NodeFusingType::FusedTerminator); +} +// todo: Skipping MultiSubGraphOp such as TensorIterator, Loop and If. Snippets might tokenize their bodies in the future. +// Note that the function is recurrent, since there might be multi-level MultiSubGraphOp, if(){if(){}}else{} for example. +void MarkSubgraphOpAsSkipped(const std::shared_ptr &node) { + if (ov::is_type(node)) { + std::vector> models{}; + // Covers TensorIterator and Loop + if (auto s = ov::as_type_ptr(node)) { + models.push_back(s->get_function()); + // Add new multi-body subgraph op here + } else if (auto if_op = ov::as_type_ptr(node)) { + models.push_back(if_op->get_then_body()); + models.push_back(if_op->get_else_body()); + } + for (auto& m : models) { + for (auto& n : m->get_ops()) { + snippets::pass::SetSnippetsNodeType(n, snippets::pass::SnippetsNodeType::SkippedByPlugin); + MarkSubgraphOpAsSkipped(n); + } + } + } +} +} // namespace + +bool SnippetsMarkSkipped::run_on_model(const std::shared_ptr &m) { + for (auto &node : m->get_ordered_ops()) { + if (ngraph::op::is_constant(node)) + continue; + if (ngraph::op::is_parameter(node)) { + SetNodeFusingType(node, NodeFusingType::IgnoredAfterInputs); + continue; + } else if (isSuitableConvolutionParent(node)) { + // Initiate fusing chain + SetNodeFusingType(node, NodeFusingType::FusedWithConvolution); + continue; + } else if (isSuitableBinaryConvolutionParent(node)) { + SetNodeFusingType(node, NodeFusingType::FusedWithBinaryConvolution); + continue; + } else if (isSuitableMiscParent(node)) { + SetNodeFusingType(node, NodeFusingType::FusedWithMisc); + continue; + } else if (isSuitableMatMulParent(node)) { + SetNodeFusingType(node, NodeFusingType::FusedWithMatMul); + continue; + } + for (const auto fusingChainType : getContinuableChains(node)) { + if (isSuitableChildForFusingSimple(node)) { + PropagateIfHasOnlyChild(node, fusingChainType); + } else if (fusingChainType == NodeFusingType::FusedWithConvolution || + fusingChainType == NodeFusingType::FusedWithBinaryConvolution) { + if (isSuitableParentForFusingSumActivation(node)) { + PropagateIfHasOnlyChild(node, NodeFusingType::FusedWithConvolutionSumActivation); + // Mimic FuseConvolutionAndSimpleOperationThroughMaxPool + } else if (isSuitablePoolChild(node)) { + PropagateIfHasOnlyChild(node, fusingChainType); + } + } else if (fusingChainType == NodeFusingType::FusedWithConvolutionSumActivation && + isSuitableChildForFusingSumActivation(node)) { + // Todo: Chain could be converted from FusedWithBinaryConvolution to FusedWithConvolution at this point + // Set FusedWithConvolution, so the fusing chain could be propagated + PropagateIfHasOnlyChild(node, NodeFusingType::FusedWithConvolution); + } else if (fusingChainType == NodeFusingType::FusedWithMatMul) { + // Handle fusings for both MatMul and FullyConnected + NodeFusingType updatedChainType = fusingChainType; + if (isSuitableChildForFusingMatMul(node, updatedChainType)) + PropagateIfHasOnlyChild(node, updatedChainType); + } else if (fusingChainType == NodeFusingType::IgnoredAfterInputs && snippets::pass::AppropriateForSubgraph(node)) { + SetNodeFusingType(node, NodeFusingType::IgnoredAfterInputs); + } + } + if (GetNodeFusingType(node) != NodeFusingType::NotSet) { + SetSnippetsNodeType(node, snippets::pass::SnippetsNodeType::SkippedByPlugin); + } else { + MarkSubgraphOpAsSkipped(node); + } + } + return true; +} +} // namespace MKLDNNPlugin \ No newline at end of file diff --git a/inference-engine/src/mkldnn_plugin/ngraph_transformations/snippets_mark_skipped.hpp b/inference-engine/src/mkldnn_plugin/ngraph_transformations/snippets_mark_skipped.hpp new file mode 100644 index 00000000000..0dce74c8bf6 --- /dev/null +++ b/inference-engine/src/mkldnn_plugin/ngraph_transformations/snippets_mark_skipped.hpp @@ -0,0 +1,39 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +namespace MKLDNNPlugin { +/** + * @interface SnippetsMarkSkipped + * @brief Mark operations that should be ignored by snippets on tokenization stage. A typical example is eltwise operations + * that will be fused into convolutions on plugin side. + */ +class SnippetsMarkSkipped : public ov::pass::ModelPass { +public: + NGRAPH_RTTI_DECLARATION; + SnippetsMarkSkipped() : ModelPass() {} + bool run_on_model(const std::shared_ptr &) override; +}; + +/* +NotSet - not part of a fusing chain +FusedTerminator - the node is fused, but the chain can't be continued +FusedWithConvolution, FusedWithConvolutionSumActivation, FusedWithMisc - fusing chains with different continuation rules +IgnoredAfterInputs - node must be skipped, since can't be handled properly at this time. Also a continuable fusing chain. +Order of SnippetsNodeType is important!: +* SnippetsNodeType >= FusedTerminator is a Fused chain +* SnippetsNodeType > FusedTerminator is a Fused chain that may be continued +*/ +// Todo: Snippets currently support only FP32 precision, however network inputs could be converted to another precision by plugin +// (in other words after tokenization). To handle this behavior, eltwise chains that start at input are marked as IgnoredAfterInputs. +// Tis is not a real plugin-side fusing, but rather a workaround to guarantee that snippets always executed in FP32. +enum class NodeFusingType : int64_t { + NotSet, + FusedTerminator, + FusedWithConvolution, FusedWithBinaryConvolution, FusedWithConvolutionSumActivation, + FusedWithMatMul, FusedWithMisc, IgnoredAfterInputs}; +} // namespace MKLDNNPlugin \ No newline at end of file diff --git a/inference-engine/src/mkldnn_plugin/nodes/subgraph.cpp b/inference-engine/src/mkldnn_plugin/nodes/subgraph.cpp new file mode 100644 index 00000000000..f7341d48fea --- /dev/null +++ b/inference-engine/src/mkldnn_plugin/nodes/subgraph.cpp @@ -0,0 +1,548 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "subgraph.h" + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include "emitters/cpu_generator.hpp" + +using namespace MKLDNNPlugin; +using namespace InferenceEngine; +using namespace mkldnn::impl::utils; +using namespace mkldnn::impl::cpu; +using namespace mkldnn::impl::cpu::x64; +using namespace Xbyak; + +MKLDNNSnippetNode::MKLDNNSnippetNode(const std::shared_ptr& op, const dnnl::engine& eng, MKLDNNWeightsSharing::Ptr &cache) + : MKLDNNNode(op, eng, cache) { + host_isa = dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_common) ? + dnnl::impl::cpu::x64::avx512_common : dnnl::impl::cpu::x64::avx2; + + // Create a deep local copy of the input snippet to perform canonicalization & code generation + // Todo: Probably better to implement a proper copy constructor + if (const auto tmp_snippet = ov::as_type_ptr(op)) { + ngraph::OutputVector subgraph_node_inputs; + for (const auto &input : tmp_snippet->input_values()) { + auto new_input = std::make_shared(input.get_element_type(), input.get_partial_shape()); + subgraph_node_inputs.push_back(new_input); + } + auto new_body = ov::clone_model(*tmp_snippet->get_body().get()); + snippet = std::make_shared(subgraph_node_inputs, new_body); + ngraph::copy_runtime_info(tmp_snippet, snippet); + snippet->set_friendly_name(tmp_snippet->get_friendly_name()); + snippet->set_generator(std::make_shared(host_isa)); + } else { + IE_THROW(NotImplemented) << "Node is not an instance of snippets::op::Subgraph"; + } +} + +void MKLDNNSnippetNode::initSupportedPrimitiveDescriptors() { + if (!supportedPrimitiveDescriptors.empty()) + return; + + auto hasBroadcastByC = [this]() -> bool { + for (auto op : ngraph::as_type_ptr(snippet)->get_body()->get_ops()) { + if (ngraph::op::supports_auto_broadcast(op)) { + auto shape = op->get_input_shape(0); + // Filter out scalar empty shape Shape{} + if (ngraph::shape_size(shape) != 1) { + for (const auto& input : op->inputs()) { + if (input.get_shape().size() > 1 && shape[1] != input.get_shape()[1] && ngraph::shape_size(input.get_shape()) != 1) { + return true; + } + } + } else { + return false; + } + } + } + return false; + }; + + const Precision supportedPrecision = Precision::FP32; + + bool dimRanksAreEqual = true; + for (size_t i = 0; dimRanksAreEqual && i < inputShapes.size(); i++) { + for (size_t j = 0; dimRanksAreEqual && j < outputShapes.size(); j++) { + if (inputShapes[i].getRank() != outputShapes[j].getRank()) + dimRanksAreEqual = false; + } + } + + const size_t ndims = outputShapes[0].getRank(); + const bool isChannelsFirstApplicable = dnnl::impl::utils::one_of(ndims, 1, 2, 4, 5) && dimRanksAreEqual; + // Todo: Snippets currently don't support per-channel broadcasting of Blocked descriptors because + // canonicalization can't distinguish between and cases. So we need to pass an + // additional parameter to canonicalization, see snippets::op::Subgraph::canonicalize for details. + const bool isBlockedApplicable = dnnl::impl::utils::one_of(ndims, 4, 5) && dimRanksAreEqual && !hasBroadcastByC(); + enum LayoutType { + Planar, + ChannelsFirst, + Blocked + }; + auto initDesc = [&] (LayoutType lt) -> NodeDesc { + auto createMemoryDesc = [lt](const Shape &shape, Precision prc, size_t offset) -> std::shared_ptr { + const auto &dims = shape.getDims(); + if (lt == ChannelsFirst && shape.getRank() != 1) { + auto ndims = shape.getRank(); + VectorDims order(ndims); + std::iota(order.begin(), order.end(), 0); + if (ndims > 1) { + order.erase(order.begin() + 1); + order.push_back(1); + } + + VectorDims blocks(ndims); + for (size_t i = 0; i < order.size(); i++) { + blocks[i] = dims[order[i]]; + } + + return std::make_shared(prc, shape, blocks, order, offset); + } else if (lt == Blocked && shape.getRank() != 1 && (shape.getMinDims()[1] != Shape::UNDEFINED_DIM && shape.getMinDims()[1] > 1)) { + size_t blockSize = mayiuse(dnnl::impl::cpu::x64::avx512_common) ? 16 : 8; + + VectorDims blocks = dims; + VectorDims order(blocks.size()); + std::iota(order.begin(), order.end(), 0); + + blocks[1] = dims[1] != Shape::UNDEFINED_DIM ? div_up(blocks[1], blockSize) : Shape::UNDEFINED_DIM; + blocks.push_back(blockSize); + order.push_back(1); + + return std::make_shared(prc, shape, blocks, order, offset); + } else { + VectorDims blocks = dims; + VectorDims order(blocks.size()); + std::iota(order.begin(), order.end(), 0); + + return std::make_shared(prc, shape, blocks, order, offset); + } + }; + + size_t offset = std::numeric_limits::max(); + NodeConfig config; + config.dynBatchSupport = false; + config.inConfs.resize(inputShapes.size()); + for (size_t i = 0; i < inputShapes.size(); i++) { + PortConfig portConfig; + portConfig.inPlace = (!i && canBeInPlace()) ? 0 : -1; + portConfig.constant = false; + portConfig.desc = createMemoryDesc(inputShapes[i], supportedPrecision, offset); + if (inputShapes[i].getDims()[0] == 1) { + const auto denseDesc = portConfig.desc->as(); + auto strides = denseDesc->getStrides(); + strides[0] = Shape::UNDEFINED_DIM; + portConfig.desc = std::make_shared(denseDesc->getPrecision(), + denseDesc->getShape(), + denseDesc->getBlockDims(), + denseDesc->getOrder(), + denseDesc->getOffsetPadding(), + denseDesc->getOffsetPaddingToData(), + strides); + } + config.inConfs[i] = portConfig; + } + config.outConfs.resize(outputShapes.size()); + for (size_t i = 0; i < outputShapes.size(); i++) { + PortConfig portConfig; + portConfig.inPlace = -1; + portConfig.constant = false; + portConfig.desc = createMemoryDesc(outputShapes[i], supportedPrecision, offset); + if (outputShapes[i].getDims()[0] == 1) { + const auto denseDesc = portConfig.desc->as(); + auto strides = denseDesc->getStrides(); + strides[0] = Shape::UNDEFINED_DIM; + portConfig.desc = std::make_shared(denseDesc->getPrecision(), + denseDesc->getShape(), + denseDesc->getBlockDims(), + denseDesc->getOrder(), + denseDesc->getOffsetPadding(), + denseDesc->getOffsetPaddingToData(), + strides); + } + config.outConfs[i] = portConfig; + } + + impl_desc_type impl_type = impl_desc_type::unknown; + if (mayiuse(x64::avx512_common)) { + impl_type = impl_desc_type::jit_avx512; + } else if (mayiuse(x64::avx2)) { + impl_type = impl_desc_type::jit_avx2; + } + return {config, impl_type}; + }; + + if (isChannelsFirstApplicable) + supportedPrimitiveDescriptors.emplace_back(initDesc(ChannelsFirst)); + if (isBlockedApplicable) + supportedPrimitiveDescriptors.emplace_back(initDesc(Blocked)); + supportedPrimitiveDescriptors.emplace_back(initDesc(Planar)); +} + +void MKLDNNSnippetNode::selectOptimalPrimitiveDescriptor() { + selectPreferPrimitiveDescriptor(getPrimitivesPriority(), true); +} + +void MKLDNNSnippetNode::createPrimitive() { + // schedule definition part + // it defines offsets, strides and sizes for snippet kernel scheduling + define_schedule(); + + // code generation part + // it might be worth to generate explicitly for scheduler work amount for now, + // but in future some interface should be defined in order to communicate schedule for a kernel + // or generate schedule for a kernel. + // Here kernel is generated for most warying dimension by default. + generate(); +} + +void MKLDNNSnippetNode::execute(dnnl::stream strm) { + if (schedule.ptr == nullptr || !canUseOptimizedImpl) { + IE_THROW() << "MKLDNNSnippetNode can't use Optimized implementation and can't fallback to reference"; + } + jit_snippets_call_args call_args; + for (size_t i = 0; i < srcMemPtrs.size(); i++) + call_args.src_ptrs[i] = reinterpret_cast(srcMemPtrs[i]->GetData()) + start_offset_in[i]; + + for (size_t i = 0; i < dstMemPtrs.size(); i++) + call_args.dst_ptrs[i] = reinterpret_cast(dstMemPtrs[i]->GetData()) + start_offset_out[i]; + + if (tensorRank == rank6D) { + schedule_6d(call_args); + } else { + schedule_nt(call_args); + } +} + +bool MKLDNNSnippetNode::created() const { + return getType() == Subgraph; +} + +// internal interface for subgraph execution + +static size_t argmax_rank(const std::vector &childEdges) { + auto getOutBlockedDims = [childEdges](int i) { + if ( auto childEdge = childEdges[i].lock() ) + return childEdge->getMemory().GetDescWithType()->getBlockDims(); + else + IE_THROW() << "Unable to lock childEdge weak_ptr"; + return VectorDims{}; + }; + auto getOutRank = [getOutBlockedDims](int i) { + return getOutBlockedDims(i).size(); + }; + size_t max_rank_idx = 0; + size_t max_rank_val = getOutRank(0); + for (size_t i = 1; i < childEdges.size(); i++) { + const auto i_rank_val = getOutRank(i); + if (max_rank_val < i_rank_val) { + max_rank_idx = i; + max_rank_val = i_rank_val; + } else if (max_rank_val == i_rank_val) { + const auto max_rank_dims = getOutBlockedDims(max_rank_idx); + const auto i_dims = getOutBlockedDims(i); + for (size_t j = 0; j < max_rank_val; j++) { + if (i_dims[j] > max_rank_dims[j]) { + max_rank_idx = i; + max_rank_val = i_rank_val; + break; + } + } + } + } + return max_rank_idx; +} + +static auto offset_calculation(std::vector& offset, const std::vector& dims_in, const std::vector& dims_out) -> void { + int k = 1; + for (int i = offset.size() - 1; i >= 0; i--) { + offset[i] = (dims_in[i] == dims_out[i]) ? k : 0; + k *= dims_in[i]; + } +} + +static auto collapseLastDims(std::vector& dims, int dimsToCollapse) -> void { + for (int i = dims.size() - 2; i > dims.size() - dimsToCollapse - 2; i--) { + dims[dims.size() - 1] *= dims[i]; + } + + for (int i = dims.size() - 2; i >= dimsToCollapse; i--) { + dims[i] = dims[i - dimsToCollapse]; + } + + for (int i = dimsToCollapse - 1; i >= 0; i--) { + dims[i] = 1; + } +} + +void MKLDNNSnippetNode::define_schedule() { + const auto config = getSelectedPrimitiveDescriptor()->getConfig(); + const auto dataSize = config.inConfs[0].desc->getPrecision().size(); + // store to use as an execution domain + max_rank_out_desc_idx = argmax_rank(getChildEdges()); + const auto outBlockingDesc_maxRank = getChildEdgeAt(max_rank_out_desc_idx)->getMemory().GetDescWithType(); + // initialize by maximum output dimension. Dimensions of outputs should be broadcastable + tensorRank = std::max(static_cast(rank6D), outBlockingDesc_maxRank->getBlockDims().size()); + + auto initDims = [this, config, &outBlockingDesc_maxRank](size_t tensorRank) { + // assume all input sizes are even + const size_t inputNum = getParentEdges().size(); + + dims_in.resize(inputNum); + for (size_t i = 0; i < inputNum; i++) { + dims_in[i].resize(tensorRank, 1); + } + + const auto outOrder = outBlockingDesc_maxRank->getOrder(); + for (size_t i = 0; i < inputNum; i++) { + auto inBlockingDesc = getParentEdgeAt(i)->getMemory().GetDescWithType(); + size_t rank = inBlockingDesc->getBlockDims().size(); + + // WA to normalize blocked and planar layouts + // not actual thought, since [§] doesn't support mixed layouts yet + auto inOrder = inBlockingDesc->getOrder(); + size_t startOff = outOrder.size() != outBlockingDesc_maxRank->getShape().getRank() && + outOrder.back() != inOrder.back() ? 1 : 0; + for (size_t j = 0; j < rank; j++) { + dims_in[i][dims_in[i].size() - 1 - j - startOff] = inBlockingDesc->getBlockDims()[rank - 1 - j]; + } + } + + // assume all output sizes are even + const size_t outputNum = config.outConfs.size(); + + dims_out.resize(outputNum); + for (size_t i = 0; i < outputNum; i++) { + dims_out[i].resize(tensorRank, 1); + } + + for (size_t i = 0; i < outputNum; i++) { + auto outBlockingDesc = getChildEdgeAt(i)->getMemory().GetDescWithType(); + size_t rank = outBlockingDesc->getBlockDims().size(); + + for (size_t j = 0; j < rank; j++) { + dims_out[i][dims_out[i].size() - 1 - j] = outBlockingDesc->getBlockDims()[rank - 1 - j]; + } + } + }; + + auto initOffsets = [this, config, dataSize](size_t tensorRank) { + // find max rank input among all outputs + const size_t inputNum = getParentEdges().size(); + offsets_in.resize(inputNum); + for (size_t i = 0; i < inputNum; i++) { + offsets_in[i].resize(tensorRank, 1); + offset_calculation(offsets_in[i], dims_in[i], dims_out[max_rank_out_desc_idx]); + for (size_t j = 0; j < tensorRank; j++) { + offsets_in[i][j] *= dataSize; + } + } + + start_offset_in.resize(inputNum); + srcMemPtrs.resize(inputNum); + for (size_t i = 0; i < inputNum; i++) { + const auto memPtr = getParentEdgeAt(i)->getMemoryPtr(); + srcMemPtrs[i] = memPtr; + start_offset_in[i] = memPtr->GetDescWithType()->getOffsetPadding() * dataSize; + } + + const size_t outputNum = config.outConfs.size(); + offsets_out.resize(outputNum); + for (size_t i = 0; i < outputNum; i++) { + offsets_out[i].resize(tensorRank, 1); + offset_calculation(offsets_out[i], dims_out[i], dims_out[max_rank_out_desc_idx]); + for (size_t j = 0; j < tensorRank; j++) { + offsets_out[i][j] *= dataSize; + } + } + + start_offset_out.resize(outputNum); + dstMemPtrs.resize(outputNum); + for (size_t i = 0; i < outputNum; i++) { + const auto memPtr = getChildEdgeAt(i)->getMemoryPtr(); + dstMemPtrs[i] = memPtr; + start_offset_out[i] = memPtr->GetDescWithType()->getOffsetPadding() * dataSize; + } + }; + + auto find_dims_to_collapse = [this, config, &outBlockingDesc_maxRank]() -> int { + int collapsedDims = 0; + size_t minimalConcurrency = parallel_get_max_threads(); + size_t minimalJitWorkAmount = 256; + size_t currentJitWorkAmount = dims_out[max_rank_out_desc_idx].back(); + while (currentJitWorkAmount < minimalJitWorkAmount && currentJitWorkAmount < fullWorkAmount) { + if (dims_out[max_rank_out_desc_idx].size() - collapsedDims - 2 < 0) + break; + + bool canCollapse = true; + for (size_t i = 0; i < dims_in.size(); i++) { + if ((dims_in[i][dims_in[i].size() - 2] != 1 && dims_in[i][dims_in[i].size() - 1] == 1) || + (dims_in[i][dims_in[i].size() - 2] == 1 && dims_in[i][dims_in[i].size() - 1] != 1)) { + canCollapse = false; + break; + } + } + + size_t nextJitWorkAmount = currentJitWorkAmount * dims_out[max_rank_out_desc_idx][dims_out[max_rank_out_desc_idx].size() - 2]; + if (fullWorkAmount / nextJitWorkAmount >= minimalConcurrency) { + currentJitWorkAmount = nextJitWorkAmount; + // if we cannot use dim collapsing we should use tile2D + if (!canCollapse) { + if (tileRank < maxTileRank) { + tileRank++; + continue; + } + + break; + } + + collapsedDims++; + for (size_t i = 0; i < dims_in.size(); i++) { + collapseLastDims(dims_in[i], 1); + } + + for (size_t i = 0; i < dims_out.size(); i++) { + collapseLastDims(dims_out[i], 1); + } + } else { + break; + } + } + return collapsedDims; + }; + + auto initSchedulingInfo = [this, dataSize](const size_t tensorRank) -> void { + // initialize scheduling information + sch_offsets_in.resize(offsets_in.size(), 0); + sch_offsets_out.resize(offsets_out.size(), 0); + sch_dims.resize(maxTileRank, 1); + sch_dims[maxTileRank-1] = dims_out[max_rank_out_desc_idx].back(); + schedulerWorkAmount = fullWorkAmount / dims_out[max_rank_out_desc_idx].back(); + if (tileRank > 1) { + sch_dims[maxTileRank - tileRank] = dims_out[max_rank_out_desc_idx][tensorRank - 2]; + schedulerWorkAmount /= dims_out[max_rank_out_desc_idx][tensorRank - 2]; + dims_out[max_rank_out_desc_idx][tensorRank - 2] = 1; + + // update offsets for tile 2D because loaders have ptr shifts in some cases and stores have always ptrs shifts + for (size_t i = 0; i < offsets_in.size(); i++) { + int64_t offset = offsets_in[i][tensorRank - 2]; + if ((offset > dataSize) || (offset == 0 && dims_in[i].back() != 1)) { + sch_offsets_in[i] = offset - dims_out[max_rank_out_desc_idx].back() * dataSize; + } else if (offset == dataSize) { + sch_offsets_in[i] = offset; + } + } + + for (size_t i = 0; i < offsets_out.size(); i++) { + int64_t offset = offsets_out[i][tensorRank - 2]; + sch_offsets_out[i] = offset - dims_out[max_rank_out_desc_idx].back() * dataSize; + } + } + }; + + initDims(tensorRank); + + fullWorkAmount = 1; + for (size_t i = 0; i < dims_out[max_rank_out_desc_idx].size(); i++) { + fullWorkAmount *= dims_out[max_rank_out_desc_idx][i]; + } + + const int collapsedDims = find_dims_to_collapse(); + batchDimIdx = tensorRank - outBlockingDesc_maxRank->getBlockDims().size() + collapsedDims; + + initOffsets(tensorRank); + initSchedulingInfo(tensorRank); +} + +void MKLDNNSnippetNode::generate() { + std::vector input_first_row; + for (size_t i = 0; i < inputShapes.size(); i++) + input_first_row.push_back(getParentEdgesAtPort(i)[0]); + + auto edgeToBlockedShape = [](const MKLDNNEdgePtr& edge) -> ngraph::snippets::op::Subgraph::BlockedShape { + const auto blockedDesc = edge->getMemory().GetDescWithType(); + ngraph::Shape shape(blockedDesc->getBlockDims()); + ngraph::AxisVector blocking(blockedDesc->getOrder()); + ngraph::element::Type precision = InferenceEngine::details::convertPrecision(blockedDesc->getPrecision()); + return std::make_tuple(shape, blocking, precision); + }; + ngraph::snippets::op::Subgraph::BlockedShapeVector input_blocked_shapes; + std::transform(input_first_row.begin(), input_first_row.end(), std::back_inserter(input_blocked_shapes), edgeToBlockedShape); + + std::vector output_first_row; + for (size_t i = 0; i < outputShapes.size(); i++) + // Can it go with difference shape or precision to different edges? I assume no. + output_first_row.push_back(getChildEdgesAtPort(i)[0]); + + ngraph::snippets::op::Subgraph::BlockedShapeVector output_blocked_shapes; + std::transform(output_first_row.begin(), output_first_row.end(), std::back_inserter(output_blocked_shapes), edgeToBlockedShape); + jit_snippets_compile_args jcp; + jcp.output_dims = dims_out[max_rank_out_desc_idx]; + std::copy(sch_dims.begin(), sch_dims.end(), jcp.scheduler_dims); + std::copy(sch_offsets_in.begin(), sch_offsets_in.end(), jcp.scheduler_offsets); + std::copy(sch_offsets_out.begin(), sch_offsets_out.end(), &jcp.scheduler_offsets[sch_offsets_in.size()]); + size_t harness_num_dims = jcp.output_dims.size() - 1; + if (harness_num_dims > SNIPPETS_MAX_HARNESS_DIMS) { + canUseOptimizedImpl = false; + harness_num_dims = SNIPPETS_MAX_HARNESS_DIMS; + } + for (size_t i = 0; i < inputShapes.size(); i++) { + auto b = offsets_in[i].begin(); + std::copy(b, b + harness_num_dims, &jcp.data_offsets[i * harness_num_dims]); + } + for (size_t i = 0; i < outputShapes.size(); i++) { + auto b = offsets_out[i].begin(); + std::copy(b, b + harness_num_dims, &jcp.data_offsets[(inputShapes.size() + i) * harness_num_dims]); + } + schedule = snippet->generate(output_blocked_shapes, input_blocked_shapes, reinterpret_cast(&jcp)); +} + +void MKLDNNSnippetNode::schedule_6d(const jit_snippets_call_args& call_args) const { + const auto& dom = dims_out[max_rank_out_desc_idx]; + // < N, C, H, W > < 1, 1, N, C*H*W> + parallel_for5d(dom[0], dom[1], dom[2], dom[3], dom[4], + [&](int64_t d0, int64_t d1, int64_t d2, int64_t d3, int64_t d4) { + int64_t indexes[] = {d0, d1, d2, d3, d4}; + schedule.get_callable()(indexes, &call_args); + }); +} + +void MKLDNNSnippetNode::schedule_nt(const jit_snippets_call_args& call_args) const { + const auto& work_size = dims_out[max_rank_out_desc_idx]; + parallel_nt(0, [&](const int ithr, const int nthr) { + size_t start = 0, end = 0; + splitter(schedulerWorkAmount, nthr, ithr, start, end); + + std::vector indexes(work_size.size() - 1, 0); + for (size_t iwork = start; iwork < end; ++iwork) { + size_t tmp = iwork; + for (ptrdiff_t j = work_size.size() - 2; j >= 0; j--) { + indexes[j] = tmp % work_size[j]; + tmp /= work_size[j]; + } + + schedule.get_callable()(indexes.data(), &call_args); + } + }); +} + +REG_MKLDNN_PRIM_FOR(MKLDNNSnippetNode, Subgraph); + diff --git a/inference-engine/src/mkldnn_plugin/nodes/subgraph.h b/inference-engine/src/mkldnn_plugin/nodes/subgraph.h new file mode 100644 index 00000000000..a434c80d0f6 --- /dev/null +++ b/inference-engine/src/mkldnn_plugin/nodes/subgraph.h @@ -0,0 +1,91 @@ +// Copyright (C) 2020-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include +#include +#include "emitters/jit_snippets_emitters.hpp" + +#include "mkldnn_node.h" +#include "snippets/op/subgraph.hpp" + +#include + +namespace MKLDNNPlugin { + +/// MKLDNNSnippetNode represents subgraph node in MKLDNN plugin +/// potentially, snippet can be placed as a postop to any support operation while it doesn't support postops itself +/// precision: fp32 +class MKLDNNSnippetNode : public MKLDNNNode { +public: + MKLDNNSnippetNode(const std::shared_ptr& op, const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache); + ~MKLDNNSnippetNode() override = default; + + void getSupportedDescriptors() override {}; + void initSupportedPrimitiveDescriptors() override; + void selectOptimalPrimitiveDescriptor() override; + + // Here we convert to canonical for & jit everything + void createPrimitive() override; + + bool created() const override; + + // if generator is set, it would execute generated code otherwise it would fallback to nGraph reference + void execute(mkldnn::stream strm) override; + +private: + static const size_t rank6D {6}; + + typedef void (*kernel)(const void *, const void *); + + void define_schedule(); + + void generate(); + + // Evaluates generated snippet using parallel backend + void schedule_6d(const jit_snippets_call_args& const_args) const; + void schedule_nt(const jit_snippets_call_args& const_args) const; + + // Local copy of subgraph node for canonization & code generation + std::shared_ptr snippet; + + // Holds generated snippet with information about how to schedule it + ngraph::snippets::Schedule schedule; + + // Holds ISA version used is codeGeneration target + dnnl::impl::cpu::x64::cpu_isa_t host_isa; + + // Holds index of output used as in execution domain + // it should be compatible with a schedule's work size + size_t max_rank_out_desc_idx = 0; + + /// scheduling info + size_t batchDimIdx = 0; + size_t tensorRank = 0; + size_t tileRank = 1; + size_t fullWorkAmount = 0; + size_t schedulerWorkAmount = 0; + const size_t maxTileRank = 2; + + std::vector srcMemPtrs = {}; + std::vector dstMemPtrs = {}; + + std::vector> dims_in = {}; + std::vector> offsets_in = {}; + std::vector start_offset_in = {}; + std::vector start_offset_out = {}; + + std::vector> dims_out = {}; + std::vector> offsets_out = {}; + + std::vector sch_dims = {}; + std::vector sch_offsets_in = {}; + std::vector sch_offsets_out = {}; + bool canUseOptimizedImpl = true; +}; + +} // namespace MKLDNNPlugin diff --git a/src/bindings/python/tests/test_onnx/test_zoo_models.py b/src/bindings/python/tests/test_onnx/test_zoo_models.py index 9530e73be2e..d012e9ced3a 100644 --- a/src/bindings/python/tests/test_onnx/test_zoo_models.py +++ b/src/bindings/python/tests/test_onnx/test_zoo_models.py @@ -54,6 +54,7 @@ tolerance_map = { "fp16_inception_v1": {"atol": 0.001, "rtol": 0.001}, "mobilenet_opset7": {"atol": 0.001, "rtol": 0.001}, "resnet50_v2_opset7": {"atol": 0.001, "rtol": 0.001}, + "resnet50-v2-7": {"atol": 0.001, "rtol": 0.001}, "test_mobilenetv2-1.0": {"atol": 0.001, "rtol": 0.001}, "test_resnet101v2": {"atol": 0.001, "rtol": 0.001}, "test_resnet18v2": {"atol": 0.001, "rtol": 0.001}, @@ -97,7 +98,7 @@ tolerance_map = { "tiny_yolov2": {"atol": 1e-05, "rtol": 0.001}, "mobilenetv2-1": {"atol": 1e-04, "rtol": 0.001}, "resnet101v1": {"atol": 1e-04, "rtol": 0.001}, - "resnet101v2": {"atol": 1e-06, "rtol": 0.001}, + "resnet101v2": {"atol": 1e-5, "rtol": 0.001}, "resnet152v2": {"atol": 1e-05, "rtol": 0.001}, "resnet18v2": {"atol": 1e-05, "rtol": 0.001}, "resnet34v2": {"atol": 1e-05, "rtol": 0.001}, diff --git a/src/common/snippets/CMakeLists.txt b/src/common/snippets/CMakeLists.txt index e9105c9044f..de6b5dd3992 100644 --- a/src/common/snippets/CMakeLists.txt +++ b/src/common/snippets/CMakeLists.txt @@ -25,10 +25,10 @@ ie_faster_build(${TARGET_NAME} UNITY ) -target_link_libraries(${TARGET_NAME} PUBLIC ngraph +target_link_libraries(${TARGET_NAME} PUBLIC ngraph inference_engine_transformations PRIVATE ngraph_reference) -target_include_directories(${TARGET_NAME} PUBLIC ${PUBLIC_HEADERS_DIR} +target_include_directories(${TARGET_NAME} PUBLIC $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME}) @@ -36,10 +36,8 @@ add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME}) ie_mark_target_as_cc(${TARGET_NAME}) # LTO - set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) -# install -# TODO: uncomment once snippets are integrated into CPU plugin -# ov_install_static_lib(${TARGET_NAME} core) +# install +ov_install_static_lib(${TARGET_NAME} core) diff --git a/src/common/snippets/include/snippets/generator.hpp b/src/common/snippets/include/snippets/generator.hpp index 9943b60a644..a8a79c91c19 100644 --- a/src/common/snippets/include/snippets/generator.hpp +++ b/src/common/snippets/include/snippets/generator.hpp @@ -3,7 +3,7 @@ // /** - * @brief A file contains public interface for target indepenent code generator. + * @brief A file contains public interface for target independent code generator. * @file generator.hpp */ #pragma once @@ -112,10 +112,10 @@ public: virtual ~Generator() = default; /** * @brief virtual method any specific implementation should implement - * @param f runction in canonical for for table-based code generation + * @param m model in canonical for for table-based code generation * @return pointer to generated code */ - code generate(std::shared_ptr& f) const; + code generate(std::shared_ptr& m, const void* compile_params = nullptr) const; protected: std::shared_ptr target; diff --git a/src/common/snippets/src/itt.hpp b/src/common/snippets/include/snippets/itt.hpp similarity index 96% rename from src/common/snippets/src/itt.hpp rename to src/common/snippets/include/snippets/itt.hpp index cc4589a9008..cc974f7ed44 100644 --- a/src/common/snippets/src/itt.hpp +++ b/src/common/snippets/include/snippets/itt.hpp @@ -15,7 +15,7 @@ namespace ngraph { namespace pass { namespace itt { namespace domains { - OV_ITT_DOMAIN(IETransform); + OV_ITT_DOMAIN(SnippetsTransform); } // namespace domains } // namespace itt } // namespace pass diff --git a/src/common/snippets/include/snippets/op/kernel.hpp b/src/common/snippets/include/snippets/op/kernel.hpp index e10e73c97d3..9b604c83e50 100644 --- a/src/common/snippets/include/snippets/op/kernel.hpp +++ b/src/common/snippets/include/snippets/op/kernel.hpp @@ -13,7 +13,7 @@ namespace op { /** * @interface Kernel - * @brief Generated by Canonicalization and represents compute kernel legal for sheduling + * @brief Generated by Canonicalization and represents compute kernel legal for scheduling * @ingroup snippets */ class Kernel : public ngraph::op::Op { @@ -28,6 +28,7 @@ public: std::shared_ptr clone_with_new_inputs(const OutputVector& inputs) const override { return std::make_shared(region); } + const void *compile_params = nullptr; }; } // namespace op diff --git a/src/common/snippets/include/snippets/op/powerstatic.hpp b/src/common/snippets/include/snippets/op/powerstatic.hpp new file mode 100644 index 00000000000..e67ff979f03 --- /dev/null +++ b/src/common/snippets/include/snippets/op/powerstatic.hpp @@ -0,0 +1,44 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +namespace ngraph { +namespace snippets { +namespace op { + +/** + * @interface PowerStatic + * @brief Generated by Canonicalization for a spasical case of power innstruction which has constant power value + * @ingroup snippets + */ +class PowerStatic : public ov::op::util::UnaryElementwiseArithmetic { +public: + OPENVINO_OP("PowerStatic", "SnippetsOpset", ov::op::util::UnaryElementwiseArithmetic); + + PowerStatic() = default; + PowerStatic(const Output &arg, float power) : UnaryElementwiseArithmetic(arg), power(power) { + constructor_validate_and_infer_types(); + } + + std::shared_ptr clone_with_new_inputs(const OutputVector &new_args) const override { + check_new_args_count(this, new_args); + return std::make_shared(new_args.at(0), power); + } + bool visit_attributes(AttributeVisitor &visitor) override { + visitor.on_attribute("power", power); + return true; + } + float get_power() const { return power; } + +private: + float power = 0; +}; +} // namespace op +} // namespace snippets +} // namespace ngraph \ No newline at end of file diff --git a/src/common/snippets/include/snippets/op/staticpower.hpp b/src/common/snippets/include/snippets/op/staticpower.hpp deleted file mode 100644 index b1d6452b6e1..00000000000 --- a/src/common/snippets/include/snippets/op/staticpower.hpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include - -namespace ngraph { -namespace snippets { -namespace op { - -/** - * @interface PowerStatic - * @brief Generated by Canonicalization for a spasical case of power innstruction which has constant power value - * @ingroup snippets - */ -class PowerStatic : public ngraph::op::v1::Power { -public: - OPENVINO_OP("PowerStatic", "SnippetsOpset", ngraph::op::v1::Power); - - PowerStatic() : Power() { - } - - PowerStatic(const Output& arg0, - const Output& arg1, - const ngraph::op::AutoBroadcastSpec& auto_broadcast = - ngraph::op::AutoBroadcastSpec(ngraph::op::AutoBroadcastType::NUMPY)) : Power(arg0, arg1, auto_broadcast) { - NGRAPH_CHECK(!!std::dynamic_pointer_cast(arg1.get_node_shared_ptr()), "second argument must be scalar constant."); - } - - std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override { - check_new_args_count(this, new_args); - return std::make_shared(new_args.at(0), new_args.at(1), this->get_autob()); - } -}; - -} // namespace op -} // namespace snippets -} // namespace ngraph \ No newline at end of file diff --git a/src/common/snippets/include/snippets/op/subgraph.hpp b/src/common/snippets/include/snippets/op/subgraph.hpp index 92c10f9b972..262b86ad8c3 100644 --- a/src/common/snippets/include/snippets/op/subgraph.hpp +++ b/src/common/snippets/include/snippets/op/subgraph.hpp @@ -6,7 +6,7 @@ #include -#include +#include #include #include #include @@ -19,7 +19,7 @@ namespace op { /** * @interface Subgraph - * @brief An operation that is implemented by a function + * @brief An operation that is implemented by a model * @ingroup snippets */ class Subgraph : public ngraph::op::Op { @@ -70,9 +70,9 @@ public: using BlockedShape = std::tuple; using BlockedShapeVector = std::vector; - Subgraph(const OutputVector& args, std::shared_ptr body); + Subgraph(const OutputVector& args, std::shared_ptr body); - Subgraph(const NodeVector& args, std::shared_ptr body); + Subgraph(const NodeVector& args, std::shared_ptr body); bool visit_attributes(AttributeVisitor& visitor) override; @@ -80,7 +80,7 @@ public: std::shared_ptr clone_with_new_inputs(const OutputVector& inputs) const override; - std::shared_ptr get_body() const { + std::shared_ptr get_body() const { return m_body; } @@ -91,13 +91,11 @@ public: std::shared_ptr make_canonical_from_this(); snippets::Schedule generate(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes, - ngraph::pass::Manager opt = ngraph::pass::Manager()); - OPENVINO_SUPPRESS_DEPRECATED_START - bool evaluate(const HostTensorVector& output_values, const HostTensorVector& input_values) const override; - OPENVINO_SUPPRESS_DEPRECATED_END - + ngraph::pass::Manager opt = ngraph::pass::Manager(), const void* compile_params = nullptr); + snippets::Schedule generate(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes, + const void* compile_params = nullptr); /// Set a new body for the op; body needs to satisfy requirements on inputs/outputs - void set_body(std::shared_ptr body); + void set_body(std::shared_ptr body); // plugin sets generator for a snippet to some specific generator. // it's going to be replaced with Jitters table later @@ -114,7 +112,7 @@ private: void canonicalize(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes); void convert_to_snippet_dialect(); - std::shared_ptr m_body; + std::shared_ptr m_body; std::shared_ptr m_generator; }; @@ -124,21 +122,21 @@ static inline std::ostream& operator<<(std::ostream& os, const op::Subgraph::Blo } static inline auto is_scalar_constant(const std::shared_ptr& source_output_node) -> bool { - return !!ngraph::as_type_ptr(source_output_node) && - (source_output_node->get_shape() == ngraph::Shape() || ngraph::shape_size(source_output_node->get_shape()) == 1); + return ngraph::is_type(source_output_node) && ngraph::shape_size(source_output_node->get_shape()) == 1; }; static inline auto create_body(std::string name, const ngraph::ResultVector& results, const ngraph::ParameterVector& parameters) -> - std::shared_ptr { - auto body = std::make_shared(results, parameters, name); + std::shared_ptr { + auto body = std::make_shared(results, parameters, name); return body; }; -static inline auto build_subgraph(const std::shared_ptr& node, const ngraph::OutputVector& inputs, const std::shared_ptr& body) +static inline auto build_subgraph(const std::shared_ptr& node, const ngraph::OutputVector& inputs, + const std::shared_ptr& body, const std::string name = "") -> std::shared_ptr{ auto subgraph = std::make_shared(inputs, body); copy_runtime_info(node, subgraph); - subgraph->set_friendly_name(node->get_friendly_name()); + subgraph->set_friendly_name(name.empty() ? node->get_friendly_name() : name); return subgraph; }; diff --git a/src/common/snippets/include/snippets/op/tile.hpp b/src/common/snippets/include/snippets/op/tile.hpp index 15656ba764e..3aa477e710b 100644 --- a/src/common/snippets/include/snippets/op/tile.hpp +++ b/src/common/snippets/include/snippets/op/tile.hpp @@ -27,6 +27,7 @@ public: std::shared_ptr clone_with_new_inputs(const OutputVector& inputs) const override { return std::make_shared(region); } + const void *compile_params; }; } // namespace op diff --git a/src/common/snippets/include/snippets/pass/assign_registers.hpp b/src/common/snippets/include/snippets/pass/assign_registers.hpp index da61cd2eacb..362936256e5 100644 --- a/src/common/snippets/include/snippets/pass/assign_registers.hpp +++ b/src/common/snippets/include/snippets/pass/assign_registers.hpp @@ -21,7 +21,7 @@ public: AssignRegisters() { set_property(ngraph::pass::PassProperty::REQUIRE_STATIC_SHAPE, true); } - bool run_on_model(const std::shared_ptr& m) override; + bool run_on_model(const std::shared_ptr& m) override; }; } // namespace pass diff --git a/src/common/snippets/include/snippets/pass/collapse_subgraph.hpp b/src/common/snippets/include/snippets/pass/collapse_subgraph.hpp index e26e28b00b8..dd41f0af411 100644 --- a/src/common/snippets/include/snippets/pass/collapse_subgraph.hpp +++ b/src/common/snippets/include/snippets/pass/collapse_subgraph.hpp @@ -12,32 +12,32 @@ namespace ngraph { namespace snippets { namespace pass { +/* + NotSet - default value returned by GetSnippetsNodeType(...) if the node wasn't marked + SkippedByPlugin - indicate that snippets can't include this node in subgraph. Can be set by Plugin via SetSnippetsNodeType(...). + */ +enum class SnippetsNodeType : int64_t {NotSet, SkippedByPlugin}; +void SetSnippetsNodeType(const std::shared_ptr&, SnippetsNodeType); +SnippetsNodeType GetSnippetsNodeType(const std::shared_ptr&); +void SetTopologicalOrder(const std::shared_ptr&, int64_t); +int64_t GetTopologicalOrder(const std::shared_ptr&); +bool AppropriateForSubgraph(const std::shared_ptr&); /** - * @interface StartSubgraph - * @brief Matches multiple output loyout-oblivious operations to start a new subgraph + * @interface EnumerateNodes + * @brief Snippets rely on topological order to avoid creating cyclic dependencies. This transformation sets the order. * @ingroup snippets */ -class StartSubgraph: public ngraph::pass::MatcherPass { +class EnumerateNodes : public ov::pass::ModelPass { public: NGRAPH_RTTI_DECLARATION; - explicit StartSubgraph(bool tokenize_by_node = false); -}; - -/** - * @interface AttachToSubgraph - * @brief Matches loyout-oblivious operations with subgraph operation as an input to attech this node into it - * @ingroup snippets - */ -class AttachToSubgraph: public ngraph::pass::MatcherPass { -public: - NGRAPH_RTTI_DECLARATION; - explicit AttachToSubgraph(bool tokenize_by_node = false); + EnumerateNodes() : ModelPass() {} + bool run_on_model(const std::shared_ptr&) override; }; /** * @interface TokenizeSnippets - * @brief Splits function to subgraphs if possible using rules above + * @brief Splits model to subgraphs if possible using rules above * This pass tokenizes topology graph into subgraphs. * Those subgraphs consists of unary or binary layout-oblivious (LO) opetations found in subset 1. * Non-layout-oblivious (NLO) operations operations (called also support in this context) are ignored and become a fullstop in tokenization routine @@ -51,20 +51,16 @@ public: * 1. outputs are collected subgraph (outputs consumed by some other node & subgraph outputs consumed by the node to be merged) * 1. finally current node is replaced with the new subgraph. We cannot use replace_node because multiple nodes are replaced so * make the replacement manually by redirecting ports - * Input subgraph is prefented from visiting twice if more than one output of it consumed by currently considered node * New subgraph is introduced, if there is a loop introduced * New subgraph is introduced, if number of inputs and outputs exceeds 7 due to scheduling limitation * New subgraph is introduced, if multiple outputs of merged nodes are not broadcastable to each other (equality of all outputs is too much on the other hand) * Scalar constants are placed as is into subgraph due to optimization purpose * @ingroup snippets */ -class TokenizeSnippets: public ngraph::pass::GraphRewrite { +class TokenizeSnippets: public ngraph::pass::MatcherPass { public: NGRAPH_RTTI_DECLARATION; - TokenizeSnippets(bool tokenize_by_node = false) { - add_matcher(tokenize_by_node); - add_matcher(tokenize_by_node); - } + explicit TokenizeSnippets(); }; } // namespace pass diff --git a/src/common/snippets/include/snippets/pass/insert_load_store.hpp b/src/common/snippets/include/snippets/pass/insert_load_store.hpp index c01726fd880..d32c5e90e78 100644 --- a/src/common/snippets/include/snippets/pass/insert_load_store.hpp +++ b/src/common/snippets/include/snippets/pass/insert_load_store.hpp @@ -14,7 +14,7 @@ namespace pass { /** * @interface InsertLoad * @brief Inserts explicit load instruction after each parameter. - * The pass is used to convert function to a canonical form for code generation + * The pass is used to convert model to a canonical form for code generation * @ingroup snippets */ class InsertLoad: public ngraph::pass::MatcherPass { @@ -25,7 +25,7 @@ public: /** * @interface InsertStore * @brief Inserts explicit store instruction before each result. - * The pass is used to convert function to a canonical form for code generation + * The pass is used to convert model to a canonical form for code generation * @ingroup snippets */ class InsertStore: public ngraph::pass::MatcherPass { diff --git a/src/common/snippets/include/snippets/pass/insert_movebroadcast.hpp b/src/common/snippets/include/snippets/pass/insert_movebroadcast.hpp index 5364781ab6e..325b5024a49 100644 --- a/src/common/snippets/include/snippets/pass/insert_movebroadcast.hpp +++ b/src/common/snippets/include/snippets/pass/insert_movebroadcast.hpp @@ -14,7 +14,7 @@ namespace pass { /** * @interface InsertMoveBroadcast * @brief Inserts explicit MoveBroadcast instruction if broadcasting by most warying dimension is needed. - * The pass is used to convert function to a canonical form for code generation + * The pass is used to convert model to a canonical form for code generation * @ingroup snippets */ class InsertMoveBroadcast: public ngraph::pass::MatcherPass { diff --git a/src/common/snippets/include/snippets/pass/load_movebroadcast_to_broadcastload.hpp b/src/common/snippets/include/snippets/pass/load_movebroadcast_to_broadcastload.hpp index 32abbcbdab9..bddc2452e78 100644 --- a/src/common/snippets/include/snippets/pass/load_movebroadcast_to_broadcastload.hpp +++ b/src/common/snippets/include/snippets/pass/load_movebroadcast_to_broadcastload.hpp @@ -14,7 +14,7 @@ namespace pass { /** * @interface LoadMoveBroadcastToBroadcastLoad * @brief Fuses consecutive Load and MoveBroadcast into a single load insctruction. - * The pass is used to convert function to a canonical form for code generation + * The pass is used to convert model to a canonical form for code generation * @ingroup snippets */ class LoadMoveBroadcastToBroadcastLoad: public ngraph::pass::MatcherPass { diff --git a/src/common/snippets/include/snippets/pass/vector_to_scalar.hpp b/src/common/snippets/include/snippets/pass/vector_to_scalar.hpp index e928d5154c2..6ee7c1bda86 100644 --- a/src/common/snippets/include/snippets/pass/vector_to_scalar.hpp +++ b/src/common/snippets/include/snippets/pass/vector_to_scalar.hpp @@ -13,8 +13,8 @@ namespace pass { /** * @interface ReplaceLoadsWithScalarLoads - * @brief Replases vector loads with scalar versions. - * The pass is used to cange alement type of function in a canonical form vector to scalar. + * @brief Replaces vector loads with scalar versions. + * The pass is used to cange element type of function in a canonical form vector to scalar. * Used for tail generation * @ingroup snippets */ @@ -25,8 +25,8 @@ public: /** * @interface ReplaceStoresWithScalarStores - * @brief Replases vector stores with scalar versions. - * The pass is used to cange alement type of function in a canonical form vector to scalar. + * @brief Replaces vector stores with scalar versions. + * The pass is used to cange element type of model in a canonical form vector to scalar. * Used for tail generation * @ingroup snippets */ diff --git a/src/common/snippets/include/snippets/snippets_isa.hpp b/src/common/snippets/include/snippets/snippets_isa.hpp index 9f4f1cd663d..be00e5e9b2a 100644 --- a/src/common/snippets/include/snippets/snippets_isa.hpp +++ b/src/common/snippets/include/snippets/snippets_isa.hpp @@ -11,13 +11,15 @@ #include "op/blockedparameter.hpp" #include "op/broadcastload.hpp" #include "op/broadcastmove.hpp" +#include "op/kernel.hpp" #include "op/load.hpp" #include "op/nop.hpp" #include "op/scalar.hpp" #include "op/scalarload.hpp" #include "op/scalarstore.hpp" -#include "op/staticpower.hpp" +#include "op/powerstatic.hpp" #include "op/store.hpp" +#include "op/tile.hpp" #include "op/vectorload.hpp" #include "op/vectorstore.hpp" diff --git a/src/common/snippets/src/generator.cpp b/src/common/snippets/src/generator.cpp index 13659752a81..1489afa89aa 100644 --- a/src/common/snippets/src/generator.cpp +++ b/src/common/snippets/src/generator.cpp @@ -8,10 +8,12 @@ #include "snippets/pass/insert_load_store.hpp" #include "snippets/op/tile.hpp" #include "snippets/op/kernel.hpp" +#include #include auto ngraph::snippets::getRegisters(std::shared_ptr& n) -> ngraph::snippets::RegInfo { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::getRegisters") auto rt = n->get_rt_info(); // ToDo: change to reg_t @@ -27,7 +29,7 @@ auto ngraph::snippets::getRegisters(std::shared_ptr& n) -> ngraph: for (auto input : n->inputs()) { auto rt = input.get_source_output().get_node_shared_ptr()->get_rt_info(); auto it_rt = rt.find("reginfo"); - if (it_rt == rt.end()) { + if (it_rt != rt.end()) { for (auto reg : it_rt->second.as>()) { rin.push_back(reg); } @@ -36,53 +38,69 @@ auto ngraph::snippets::getRegisters(std::shared_ptr& n) -> ngraph: return std::make_pair(rin, rout); } -ngraph::snippets::code ngraph::snippets::Generator::generate(std::shared_ptr& f) const { +ngraph::snippets::code ngraph::snippets::Generator::generate(std::shared_ptr& m, + const void* compile_params) const { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::Generator::generate") if (!target->is_supported()) throw ngraph_error("unsupported architecture for code genration"); - auto params = f->get_parameters(); - auto results = f->get_results(); - auto nptrs = results.size() + params.size(); - - if (nptrs > 7) { - throw ngraph_error("snippet signature should not exceed 7 arguments. got " + std::to_string(nptrs)); - } + auto params = m->get_parameters(); + auto results = m->get_results(); + auto in = params.size(); + auto out = results.size(); + auto nptrs = in + out; + OV_ITT_TASK_CHAIN(GENERATE, ngraph::pass::itt::domains::SnippetsTransform, "Snippets::Generator", "::VectorTile") // vector tile std::vector, ngraph::snippets::RegInfo>> lowered; - for (auto n : f->get_ordered_ops()) { + for (auto n : m->get_ordered_ops()) { lowered.push_back(std::make_pair(target->get(n->get_type_info())(n), ngraph::snippets::getRegisters(n))); } + OV_ITT_TASK_NEXT(GENERATE, "::ScalarTile") // scalar tile - auto f_scalar = ngraph::clone_function(*f.get()); - ngraph::pass::Manager m; - m.register_pass(); - m.register_pass(); - m.run_passes(f_scalar); - + auto m_scalar = ov::clone_model(*m.get()); + ngraph::pass::Manager mng; + mng.register_pass(); + mng.register_pass(); + mng.run_passes(m_scalar); + OV_ITT_TASK_NEXT(GENERATE, "::ScalarTile_get") std::vector, RegInfo>> scalar_lowered; - for (auto n : f_scalar->get_ordered_ops()) { + for (auto n : m_scalar->get_ordered_ops()) { scalar_lowered.push_back(std::make_pair(target->get(n->get_type_info())(n), ngraph::snippets::getRegisters(n))); } + OV_ITT_TASK_NEXT(GENERATE, "::Tiles1D") - // wrapping into tiles - std::vector, RegInfo>> tiles; - tiles.push_back(std::make_pair(target->get(ngraph::snippets::op::Tile::get_type_info_static())( - std::make_shared(lowered)), - std::make_pair(std::vector({target->get_lanes(), nptrs}), std::vector{}))); - tiles.push_back(std::make_pair(target->get(ngraph::snippets::op::Tile::get_type_info_static())( - std::make_shared(scalar_lowered)), - std::make_pair(std::vector{{1, nptrs}}, std::vector{}))); + // wrapping into tiles1D + std::vector, RegInfo>> tiles1D; + auto tile = std::make_shared(lowered); + tile->compile_params = compile_params; + tiles1D.push_back(std::make_pair(target->get(ngraph::snippets::op::Tile::get_type_info_static())(tile), + std::make_pair(std::vector({target->get_lanes(), 0, nptrs, 1}), std::vector{}))); + tile = std::make_shared(scalar_lowered); + tile->compile_params = compile_params; + tiles1D.push_back(std::make_pair(target->get(ngraph::snippets::op::Tile::get_type_info_static())(tile), + std::make_pair(std::vector{{1, target->get_lanes(), nptrs, 1}}, std::vector{}))); + OV_ITT_TASK_NEXT(GENERATE, "::Tiles2D") + // wrapping into tiles2D + std::vector, RegInfo>> tiles2D; + tile = std::make_shared(tiles1D); + tile->compile_params = compile_params; + tiles2D.push_back(std::make_pair(target->get(ngraph::snippets::op::Tile::get_type_info_static())(tile), + std::make_pair(std::vector({1, 0, nptrs, 0}), std::vector{}))); + + OV_ITT_TASK_NEXT(GENERATE, "::EmitCode") // emission - std::shared_ptr kernel = target->get(ngraph::snippets::op::Kernel::get_type_info_static())(std::make_shared(tiles)); - kernel->emit_code({params.size(), results.size()}, {}); - + auto tiles2DKernel = std::make_shared(tiles2D); + tiles2DKernel->compile_params = compile_params; + std::shared_ptr kernel = target->get(ngraph::snippets::op::Kernel::get_type_info_static())(tiles2DKernel); + kernel->emit_code({in, out}, {}); + OV_ITT_TASK_NEXT(GENERATE, "::EmitData") lowered.insert(lowered.end(), scalar_lowered.begin(), scalar_lowered.end()); for (auto& op : lowered) { op.first->emit_data(); } - + OV_ITT_TASK_NEXT(GENERATE, "::GetSnippet") return target->get_snippet(); } diff --git a/src/common/snippets/src/op/broadcastload.cpp b/src/common/snippets/src/op/broadcastload.cpp index 0b29b21214e..89ba270f23b 100644 --- a/src/common/snippets/src/op/broadcastload.cpp +++ b/src/common/snippets/src/op/broadcastload.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "itt.hpp" +#include #include "snippets/op/broadcastload.hpp" diff --git a/src/common/snippets/src/op/broadcastmove.cpp b/src/common/snippets/src/op/broadcastmove.cpp index 5a1fd4bddca..d14f53c7e82 100644 --- a/src/common/snippets/src/op/broadcastmove.cpp +++ b/src/common/snippets/src/op/broadcastmove.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "itt.hpp" +#include #include "snippets/op/broadcastmove.hpp" diff --git a/src/common/snippets/src/op/load.cpp b/src/common/snippets/src/op/load.cpp index 8ad943ebdd4..837d64dd9b0 100644 --- a/src/common/snippets/src/op/load.cpp +++ b/src/common/snippets/src/op/load.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "itt.hpp" +#include #include "snippets/op/load.hpp" diff --git a/src/common/snippets/src/op/store.cpp b/src/common/snippets/src/op/store.cpp index 99b86b164ce..066bcf7f8fd 100644 --- a/src/common/snippets/src/op/store.cpp +++ b/src/common/snippets/src/op/store.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "itt.hpp" +#include #include "snippets/op/scalarstore.hpp" diff --git a/src/common/snippets/src/op/subgraph.cpp b/src/common/snippets/src/op/subgraph.cpp index 1270edd3e3b..d9a2fc34301 100644 --- a/src/common/snippets/src/op/subgraph.cpp +++ b/src/common/snippets/src/op/subgraph.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "itt.hpp" +#include #include "remarks.hpp" #include "snippets/op/subgraph.hpp" @@ -25,21 +25,22 @@ void snippets::op::Subgraph::set_generator(std::shared_ptr body) +snippets::op::Subgraph::Subgraph(const OutputVector& args, std::shared_ptr body) : Op(args), m_body(body), m_generator(nullptr) { constructor_validate_and_infer_types(); } -snippets::op::Subgraph::Subgraph(const NodeVector& args, std::shared_ptr body) +snippets::op::Subgraph::Subgraph(const NodeVector& args, std::shared_ptr body) : Subgraph(as_output_vector(args), body) {} std::shared_ptr snippets::op::Subgraph::clone_with_new_inputs(const OutputVector& inputs) const { INTERNAL_OP_SCOPE(Subgraph); - return make_shared(inputs, ngraph::clone_function(*m_body.get())); + return make_shared(inputs, ov::clone_model(*m_body.get())); } void snippets::op::Subgraph::validate_and_infer_types() { INTERNAL_OP_SCOPE(Subgraph); + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::validate_and_infer_types") ngraph::ParameterVector old_parameters; for (auto op : m_body->get_parameters()) { old_parameters.push_back(op); @@ -65,24 +66,24 @@ bool snippets::op::Subgraph::visit_attributes(AttributeVisitor& visitor) { return true; } -auto snippets::op::Subgraph::wrap_node_as_subgraph(const std::shared_ptr& node) -> std::shared_ptr { +auto snippets::op::Subgraph::wrap_node_as_subgraph(const std::shared_ptr& node) -> std::shared_ptr { INTERNAL_OP_SCOPE(Subgraph); + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::wrap_node_as_subgraph") ngraph::ParameterVector body_parameters; ngraph::OutputVector body_inputs; ngraph::OutputVector subgraph_inputs; - for (auto input : node->inputs()) { - auto source_output = input.get_source_output(); - if (is_scalar_constant(source_output.get_node_shared_ptr())) { - body_inputs.push_back(source_output); + for (const auto& input : node->input_values()) { + if (is_scalar_constant(input.get_node_shared_ptr())) { + body_inputs.push_back(input); } else { auto parameter = std::make_shared(input.get_element_type(), input.get_partial_shape()); body_parameters.push_back(parameter); - body_parameters.back()->set_friendly_name(source_output.get_node()->get_friendly_name()); + body_parameters.back()->set_friendly_name(input.get_node()->get_friendly_name()); body_inputs.push_back(parameter->output(0)); - subgraph_inputs.push_back(source_output); + subgraph_inputs.push_back(input); } } @@ -93,6 +94,9 @@ auto snippets::op::Subgraph::wrap_node_as_subgraph(const std::shared_ptrclear_control_dependencies(); ngraph::ResultVector body_results; for (auto output : node->outputs()) { body_results.push_back(std::make_shared(body_node->output(output.get_index()))); @@ -118,7 +122,7 @@ std::shared_ptr snippets::op::Subgraph::make_canonical_f for (auto input : this->input_values()) { subgraph_node_inputs.push_back(input); } - auto new_body = ngraph::clone_function(*this->get_body().get()); + auto new_body = ov::clone_model(*this->get_body().get()); auto snippet = std::make_shared(subgraph_node_inputs, new_body); ngraph::copy_runtime_info(this->shared_from_this(), snippet); snippet->set_friendly_name(this->get_friendly_name()); @@ -129,15 +133,16 @@ std::shared_ptr snippets::op::Subgraph::make_canonical_f // We also can think of canonization as of pass to copy original subgraph and transforming it to canonical form suitable for code generation // pass actual parameters and results shapes to generate for as well as channel mapping, -// we need to distinguish between 5d tensors that represents and somehow like locked dimensions -// ngraph::AxisVector to code +// Todo: we need to distinguish between 5d tensors that represents and somehow like locked dimensions +// ngraph::AxisVector to code void snippets::op::Subgraph::canonicalize(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes) { INTERNAL_OP_SCOPE(Subgraph); + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::canonicalize") NODE_VALIDATION_CHECK(this, input_shapes.size() == m_body->get_parameters().size(), - "Number of parameters for snippet doesn't much passed to generate method: ", input_shapes.size(), " vs ", m_body->get_parameters().size(), "."); + "Number of parameters for snippet doesn't match passed to generate method: ", input_shapes.size(), " vs ", m_body->get_parameters().size(), "."); NODE_VALIDATION_CHECK(this, output_shapes.size() == m_body->get_results().size(), - "number of results for snippet doesn't much passed to generate method: ", output_shapes.size(), " vs ", m_body->get_results().size(), "."); + "number of results for snippet doesn't match passed to generate method: ", output_shapes.size(), " vs ", m_body->get_results().size(), "."); // replace only constants which are actually should be represented as scalars during code generation and probably move this step a bit later for (auto op : m_body->get_ordered_ops()) { @@ -149,18 +154,6 @@ void snippets::op::Subgraph::canonicalize(const BlockedShapeVector& output_shape } } - // repalace power with power static - for (auto op : m_body->get_ordered_ops()) { - if (auto power = ngraph::as_type_ptr(op)) { - if (ngraph::as_type_ptr(power->input(1).get_node()->shared_from_this())) { - auto power_static = std::make_shared( - power->input(0).get_source_output(), power->input(1).get_source_output(), power->get_autob()); - power_static->set_friendly_name(power->get_friendly_name()); - ngraph::copy_runtime_info(power, power_static); - ngraph::replace_node(power, power_static); - } - } - } // it should be in subgraph node to be aligned with internal and external parameter list, but adding this for testing @@ -193,6 +186,7 @@ void snippets::op::Subgraph::canonicalize(const BlockedShapeVector& output_shape void snippets::op::Subgraph::convert_to_snippet_dialect() { INTERNAL_OP_SCOPE(Subgraph); + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::convert_to_snippet_dialect") ngraph::pass::Manager manager; manager.register_pass(); manager.register_pass(); @@ -201,24 +195,54 @@ void snippets::op::Subgraph::convert_to_snippet_dialect() { manager.run_passes(m_body); } -snippets::Schedule snippets::op::Subgraph::generate(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes, - ngraph::pass::Manager opt) { +snippets::Schedule snippets::op::Subgraph::generate(const BlockedShapeVector& output_shapes, + const BlockedShapeVector& input_shapes, + const void* compile_params) { + return generate(output_shapes, input_shapes, ngraph::pass::Manager(), compile_params); +} + +snippets::Schedule snippets::op::Subgraph::generate(const BlockedShapeVector& output_shapes, + const BlockedShapeVector& input_shapes, + ngraph::pass::Manager opt, + const void* compile_params) { INTERNAL_OP_SCOPE(Subgraph); + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::generate") NGRAPH_CHECK(m_generator != nullptr, "generate is called while generator is not set"); canonicalize(output_shapes, input_shapes); + + // Todo: ngraph::pass::Manager introduces appreciable overheads, especially while used on small graphs. + // So don't wrap this transformation as a MatcherPass, but rewrite convert_to_snippet_dialect() as a + // for loop to improve first-inference time. + // replace power with power static + + for (auto op : m_body->get_ordered_ops()) { + if (ov::is_type(op) && + ov::is_type(op->get_input_node_shared_ptr(1)) && + ov::shape_size(op->get_input_shape(1)) == 1) { + auto power = ov::as_type_ptr(op); + auto scalar = ov::as_type_ptr(op->get_input_node_shared_ptr(1)); + auto value = scalar->cast_vector()[0];; + auto power_static = std::make_shared(power->input(0).get_source_output(), value); + power_static->set_friendly_name(power->get_friendly_name()); + ngraph::copy_runtime_info(power, power_static); + ngraph::replace_node(power, power_static); + } + } + + convert_to_snippet_dialect(); opt.run_passes(m_body); // generation flow snippets::pass::AssignRegisters().run_on_model(m_body); - // shedule generation should go here and be target agnostic + // schedule generation should go here and be target agnostic // actual code emission - ngraph::snippets::code ptr = m_generator->generate(m_body); + ngraph::snippets::code ptr = m_generator->generate(m_body, compile_params); - // chack that body doesnt have constants for scheduling + // check that body doesn't have constants for scheduling std::vector> constants; for (auto op : m_body->get_ordered_ops()) { if (auto constant = ov::as_type_ptr(op)) { @@ -227,7 +251,7 @@ snippets::Schedule snippets::op::Subgraph::generate(const BlockedShapeVector& ou } } } - NGRAPH_CHECK(!constants.size(), "External constants detected. Snippet is illigal for sheduling"); + NGRAPH_CHECK(!constants.size(), "External constants detected. Snippet is illigal for scheduling"); // check resulting shapes are broadcastable to each other so can be scheduled Shape work_size = m_body->output(0).get_shape(); @@ -252,13 +276,6 @@ snippets::Schedule snippets::op::Subgraph::generate(const BlockedShapeVector& ou return {work_size, false /*canBeLinearized*/, ptr}; } -bool snippets::op::Subgraph::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { - INTERNAL_OP_SCOPE(Subgraph); - OPENVINO_SUPPRESS_DEPRECATED_START - return m_body->evaluate(outputs, inputs); - OPENVINO_SUPPRESS_DEPRECATED_END -} - void snippets::op::Subgraph::print() const { INTERNAL_OP_SCOPE(Subgraph); remark(13) << "subgraph " << this->get_friendly_name() << " " @@ -285,7 +302,7 @@ void snippets::op::Subgraph::print() const { void snippets::op::Subgraph::print_statistics(bool verbose) { INTERNAL_OP_SCOPE(Subgraph); - auto getNodeInventory = [](std::shared_ptr n) -> size_t { + auto getNodeInventory = [](std::shared_ptr n) -> size_t { size_t total = 0; for (auto input : n->inputs()) { @@ -307,7 +324,7 @@ void snippets::op::Subgraph::print_statistics(bool verbose) { return total; }; - auto getFunctionInventory = [getNodeInventory](std::shared_ptr f) -> size_t { + auto getModelInventory = [getNodeInventory](std::shared_ptr f) -> size_t { size_t total = 0; for (auto op : f->get_ordered_ops()) { // Results and parameters are artificially introduced, @@ -322,7 +339,7 @@ void snippets::op::Subgraph::print_statistics(bool verbose) { return total; }; - auto countConstants = [](std::shared_ptr f) -> size_t { + auto countConstants = [](std::shared_ptr f) -> size_t { size_t count = 0; for (auto op : f->get_ordered_ops()) { count += !!ngraph::as_type_ptr(op) ? 1 : 0; @@ -338,7 +355,7 @@ void snippets::op::Subgraph::print_statistics(bool verbose) { << ";" << body->get_parameters().size() << ";" << body->get_results().size() << ";" << countConstants(body) - << ";" << getFunctionInventory(body) + << ";" << getModelInventory(body) << ";" << getNodeInventory(this->shared_from_this()) << std::endl; if (verbose) { diff --git a/src/common/snippets/src/pass/assign_registers.cpp b/src/common/snippets/src/pass/assign_registers.cpp index e71122e860d..1ed965913f2 100644 --- a/src/common/snippets/src/pass/assign_registers.cpp +++ b/src/common/snippets/src/pass/assign_registers.cpp @@ -3,7 +3,7 @@ // // #include -#include "itt.hpp" +#include #include "remarks.hpp" #include "snippets/pass/assign_registers.hpp" @@ -13,8 +13,9 @@ #include -bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr& f) { +bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr& f) { RUN_ON_FUNCTION_SCOPE(AssignRegisters); + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::AssignRegisters") int reg64_tmp_start { 8 }; // R8, R9, R10, R11, R12, R13, R14, R15 inputs+outputs+1 using Reg = size_t; auto ops = f->get_ordered_ops(); @@ -143,7 +144,7 @@ bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr for (auto n : f->get_ordered_ops()) { auto& rt = n->get_rt_info(); - // nothing to do for function signature + // nothing to do for model signature if (std::dynamic_pointer_cast(n) || std::dynamic_pointer_cast(n)) { continue; } diff --git a/src/common/snippets/src/pass/collapse_subgraph.cpp b/src/common/snippets/src/pass/collapse_subgraph.cpp index afa847b80f5..14a38a0e2c6 100644 --- a/src/common/snippets/src/pass/collapse_subgraph.cpp +++ b/src/common/snippets/src/pass/collapse_subgraph.cpp @@ -3,7 +3,7 @@ // #include "remarks.hpp" -#include "itt.hpp" +#include #include "snippets/pass/collapse_subgraph.hpp" #include "snippets/op/subgraph.hpp" @@ -11,6 +11,7 @@ #include #include #include +#include "transformations/utils/utils.hpp" #include #include @@ -18,27 +19,29 @@ #include #include #include +#include -NGRAPH_RTTI_DEFINITION(ngraph::snippets::pass::StartSubgraph, "CollapseSubgraph", 0); -NGRAPH_RTTI_DEFINITION(ngraph::snippets::pass::AttachToSubgraph, "CollapseSubgraph", 0); -NGRAPH_RTTI_DEFINITION(ngraph::snippets::pass::TokenizeSnippets, "CollapseSubgraph", 0); +NGRAPH_RTTI_DEFINITION(ngraph::snippets::pass::TokenizeSnippets, "Snippets::TokenizeSnippets", 0); +NGRAPH_RTTI_DEFINITION(ngraph::snippets::pass::EnumerateNodes, "Snippets::EnumerateNodes", 0); + +namespace ngraph { +namespace snippets { +namespace pass { -using namespace ngraph; -using namespace snippets; namespace { -auto outputs_are_not_broadcastable(const std::shared_ptr& node) -> bool { +auto outputs_are_not_broadcastable(const std::shared_ptr& node) -> bool { auto outputs = node->outputs(); - auto find_smallest_output_shape = [](const std::vector>& outputs) -> ngraph::Shape { + auto find_smallest_output_shape = [](const std::vector>& outputs) -> Shape { return std::accumulate(std::begin(outputs), std::end(outputs), ngraph::Shape(outputs.begin()->get_shape()), - [](ngraph::Shape other_shape, ngraph::Output output){ - return ngraph::shape_size(output.get_shape()) < ngraph::shape_size(other_shape) ? output.get_shape() : other_shape; + [](Shape& other_shape, const Output& output){ + return shape_size(output.get_shape()) < shape_size(other_shape) ? output.get_shape() : other_shape; }); }; auto ref_shape = find_smallest_output_shape(outputs); - auto check_shapes_broadcastable = [ref_shape](const ngraph::Output& output) -> bool { + auto check_shapes_broadcastable = [ref_shape](const Output& output) -> bool { auto other_shape = output.get_shape(); if (other_shape.size() != ref_shape.size()) { @@ -46,300 +49,316 @@ auto outputs_are_not_broadcastable(const std::shared_ptr& node) -> } return std::inner_product(std::begin(other_shape), std::end(other_shape), std::begin(ref_shape), true, - std::logical_and(), [](ngraph::Shape::value_type lsh, ngraph::Shape::value_type rsh){ + std::logical_and(), [](Shape::value_type lsh, Shape::value_type rsh){ return rsh == 1 || lsh == rsh; }); }; return std::find_if_not(std::begin(outputs), std::end(outputs), check_shapes_broadcastable) != std::end(outputs); -}; +} -auto has_cycles_of_dependencies(const std::vector>>& results, - const std::vector>& inputs) -> bool { - auto BFS_from_to = [](ngraph::Node* from, ngraph::Node* to) -> bool { - std::unordered_set visited; - std::queue stack; - stack.push(from); - - while (stack.size() > 0) { - ngraph::Node* curr = stack.front(); - visited.insert(curr); - - stack.pop(); - - if (curr != to) { - for (const auto& next : curr->get_users()) { - if (visited.count(next.get()) == 0) { - stack.push(next.get()); - } - } - } else { - return true; - } - } - return false; +auto is_layout_oblivious(const std::shared_ptr &n) -> bool { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::is_layout_oblivious") + auto is_layout_oblivious_binary = [](const std::shared_ptr &n) -> bool { + return ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n); }; - for (auto& result : results) { - for (auto& user : result) { - for (auto& input : inputs) { - auto source = input.get_source_output().get_node(); - auto containsLoop = BFS_from_to(user.get_node(), source); + auto is_layout_oblivious_unary = [](const std::shared_ptr &n) -> bool { + return ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n) + || ov::is_type(n); + }; + return is_layout_oblivious_unary(n) || is_layout_oblivious_binary(n); +} - remark(1) << "checking path from " - << user.get_node()->get_friendly_name() - << " to " << source->get_friendly_name() - << " resulted in " << containsLoop << std::endl; - - if (containsLoop) { - return true; - } +auto has_supported_in_out(const std::shared_ptr &n) -> bool { + auto supported = [](descriptor::Tensor& t) -> bool { + return t.get_element_type() == ngraph::element::f32 && + t.get_partial_shape().is_static(); + }; + const auto & inputs = n->inputs(); + const auto & outputs = n->outputs(); + // todo: Is this check necessary? Remove if not + for (const auto& out : outputs) { + for (const auto &in_out : out.get_target_inputs()) { + if (ov::is_type(in_out.get_node()->shared_from_this())) { + return false; } } } - return false; + return std::all_of(inputs.begin(), inputs.end(), [&](const Input& in) {return supported(in.get_tensor());}) && + std::all_of(outputs.begin(), outputs.end(), [&](const Output& out) {return supported(out.get_tensor());}); } -auto has_subgraph_as_input(std::shared_ptr node) -> bool { - auto inputs = node->inputs(); - for (auto input : inputs) { - auto parent = input.get_source_output().get_node_shared_ptr(); - if (!!ov::as_type_ptr(parent)) { +auto has_result_child(const std::shared_ptr &node) -> bool { + for (const auto &child : node->get_users()) { + if (ov::is_type(child)) { return true; } } return false; -}; - -auto is_lo(std::shared_ptr n) -> bool { - auto is_lob = [](std::shared_ptr n) -> bool { - using ngraph::as_type_ptr; - return !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n); - }; - - auto is_lou = [](std::shared_ptr n) -> bool { - using ngraph::as_type_ptr; - return !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) ? - || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) ? - // || !!ov::as_type_ptr(n) ? - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) ? - || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n) - // || !!ov::as_type_ptr(n) - || !!ov::as_type_ptr(n); - }; - - auto is_lot = [](std::shared_ptr n) -> bool { - using ngraph::as_type_ptr; - return false; - // return !!ov::as_type_ptr(n) // ternary with 2 constants - // || !!ov::as_type_ptr(n); // ternary with 2 constants / or DW - }; - - auto is_fq = [](std::shared_ptr n) -> bool { - using ngraph::as_type_ptr; - return false;//!!ov::as_type_ptr(n); // 4->1 - }; - - return is_lou(n) || is_lob(n) ||is_lot(n) || is_fq(n); } -auto has_supported_in_out(std::shared_ptr n) -> bool { - for (auto in : n->inputs()) { - if (in.get_tensor().get_element_type() != ngraph::element::f32) { - return false; - } - - if (in.get_partial_shape().is_dynamic()) { - return false; - } - - if (in.get_partial_shape().is_static() && in.get_shape().size() > 6) { - return false; +auto get_num_result_children(const std::shared_ptr &node) -> size_t { + size_t result = 0; + for (const auto &child : node->get_users()) { + if (ov::is_type(child)) { + result++; } } - - for (auto out : n->outputs()) { - if (out.get_tensor().get_element_type() != ngraph::element::f32) { - return false; - } - - if (out.get_partial_shape().is_dynamic()) { - return false; - } - - if (out.get_partial_shape().is_static() && out.get_shape().size() > 6) { - return false; - } - - for (auto in_out : out.get_target_inputs()) { - if (!!ov::as_type_ptr(in_out.get_node()->shared_from_this())) { - return false; - } - - if (!!ov::as_type_ptr(in_out.get_node()->shared_from_this())) { - return false; + return result; +} +// Need to update tensor name manually, since MKLDNNGraph::Replicate() looks at input.get_tensor().get_name(); +// If subgraph->get_output_size() == 1, then the name will be restored correctly from the node name +auto update_out_tensor_name(std::shared_ptr &subgraph) -> void { + bool not_set = true; + for (unsigned int i = 0; i < subgraph->get_output_size() && not_set; i++) { + for (const auto &in : subgraph->get_output_target_inputs(i)) { + if (ov::is_type(in.get_node())) { + auto out_tensor = subgraph->output(i).get_tensor_ptr(); + NGRAPH_SUPPRESS_DEPRECATED_START + if (out_tensor->get_name().empty()) { + const auto& body_result = subgraph->get_body()->get_output_op(i); + const std::string newTensorName = ngraph::op::util::get_ie_output_name( + body_result->get_input_source_output(0)); + out_tensor->set_name(newTensorName); + } + NGRAPH_SUPPRESS_DEPRECATED_END + not_set = false; + break; } } } - - return true; -}; - +} } // namespace -ngraph::snippets::pass::StartSubgraph::StartSubgraph(bool tokenize_by_node) : MatcherPass() { - MATCHER_SCOPE(StartSubgraph); - - auto has_multiple_output_edges = [](std::shared_ptr n) -> bool { - for (auto out : n->outputs()) { - if (out.get_target_inputs().size() != 1) return true; - } - - return false; - }; - - register_matcher(std::make_shared( - std::make_shared(pattern::any_input(), - [tokenize_by_node, has_multiple_output_edges](std::shared_ptr n) { - return is_lo(n) && - has_supported_in_out(n) && - (tokenize_by_node || !has_subgraph_as_input(n)) && - has_multiple_output_edges(n); - })), - [](ngraph::pattern::Matcher &m) -> bool { - auto node = m.get_match_root(); - - remark(1) << "Match root" - << node->get_friendly_name() - << " " << node - << " Creating new snippet - no input subgraphs found" << std::endl; - - auto subgraph = op::Subgraph::wrap_node_as_subgraph(node); - ngraph::replace_node(node, subgraph); - - remark(1) << "Replacement (new) done for: " - << subgraph->get_friendly_name() - << " with " << subgraph->inputs().size() - << " inputs and " << subgraph->outputs().size() - << " outputs and " << subgraph->get_body()->get_ops().size() << " ops total\n"; - return true; - }); +bool AppropriateForSubgraph(const std::shared_ptr &node) { + return is_layout_oblivious(node) && has_supported_in_out(node); } -ngraph::snippets::pass::AttachToSubgraph::AttachToSubgraph(bool tokenize_by_node) : MatcherPass() { - MATCHER_SCOPE(AttachToSubgraph); +void SetSnippetsNodeType(const std::shared_ptr &node, SnippetsNodeType nodeType) { + auto &rt = node->get_rt_info(); + rt["SnippetsNodeType"] = nodeType; +} + +SnippetsNodeType GetSnippetsNodeType(const std::shared_ptr &node) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::GetSnippetsNodeType") + auto &rt = node->get_rt_info(); + const auto rinfo = rt.find("SnippetsNodeType"); + if (rinfo == rt.end()) + return SnippetsNodeType::NotSet; + return rinfo->second.as(); +} + +void SetTopologicalOrder(const std::shared_ptr &node, int64_t order) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::SetTopologicalOrder") + auto &rt = node->get_rt_info(); + rt["TopologicalOrder"] = order; +} + +int64_t GetTopologicalOrder(const std::shared_ptr &node) { + auto &rt = node->get_rt_info(); + const auto rinfo = rt.find("TopologicalOrder"); + if (rinfo == rt.end()) + throw ngraph_error("Topological order is required, but not set."); + return rinfo->second.as(); +} + +bool EnumerateNodes::run_on_model(const std::shared_ptr &m) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::EnumerateNodes") + int64_t order = 0; + // Todo: We don't really have to set order for every node, just for subgraph parents and children would be enough + for (auto &node : m->get_ordered_ops()) { + SetTopologicalOrder(node, order++); + } + return true; +} +TokenizeSnippets::TokenizeSnippets() { + MATCHER_SCOPE(TokenizeSnippets); enum continuation_strategy { reset, abort }; - continuation_strategy strategy = continuation_strategy::abort; - - ngraph::graph_rewrite_callback continuation_callback = [strategy](ngraph::pattern::Matcher &m) -> bool { + continuation_strategy strategy = continuation_strategy::reset; + auto label = std::make_shared(pattern::any_input(), + [](const std::shared_ptr &n) { + return GetSnippetsNodeType(n) != SnippetsNodeType::SkippedByPlugin && AppropriateForSubgraph(n); + }); + ngraph::graph_rewrite_callback callback = [&, strategy](ngraph::pattern::Matcher &m) -> bool { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::CreateSubgraph_callback") auto node = m.get_match_root(); + if (transformation_callback(node)) { + return false; + } - remark(1) << "Match root " << node->get_friendly_name() << " " << node << std::endl; + remark(1) << "Match root: " << node->get_friendly_name() << " " << node << std::endl; - // inputs that are already subgraphs - std::unordered_set> input_subgraphs; - // clone bodies because we need a rollback if loop is found - std::map, std::shared_ptr> clones; + const auto getFusedNames = [](const std::shared_ptr& n) -> std::string { + auto rt_info = n->get_rt_info(); + auto it = rt_info.find("originalLayersNames"); + if (it != rt_info.end()) { + return it->second.as() + ","; + } + return ""; + }; - ParameterVector body_parameters; - OutputVector external_inputs; - OutputVector internal_inputs; + auto create_single_node_subgraph = [&](const std::shared_ptr &node) { + auto subgraph = op::Subgraph::wrap_node_as_subgraph(node); + subgraph->get_rt_info()["originalLayersNames"] = getFusedNames(node) + node->get_friendly_name(); + ngraph::replace_node(node, subgraph); + update_out_tensor_name(subgraph); + }; - auto inputs = node->inputs(); - - auto is_recurrent = [inputs](const ngraph::Output& to_find) -> bool { - for (auto in : inputs) { - if (in.get_source_output().get_node_shared_ptr() == to_find.get_node_shared_ptr()) { - return true; + auto abort_with_strategy = [&](const std::string& message_reset, + const std::string& message_abort = "", int priority = 3) { + if (strategy == continuation_strategy::reset) { + create_single_node_subgraph(node); + return true; + } else if (strategy == continuation_strategy::abort) { + if (!message_abort.empty()) { + remark(priority) << message_abort << std::endl; } } return false; }; + // inputs that are already subgraphs + std::unordered_set> input_subgraphs; + // clone bodies because we need a rollback if loop is found + std::map, std::shared_ptr> clones; - auto get_input_index = [](const Output& found) -> size_t { - for (auto& input : found.get_target_inputs()) { - remark(13) << input.get_node() << " " << input.get_source_output() << " vs " - << found << found.get_node() << " : " << input.get_index() << " " << found.get_index() << std::endl; - } + ParameterVector body_parameters; + // inputs to merged subgraph + OutputVector external_inputs; + // inputs to the node before merge to subgraph + OutputVector internal_inputs; - for (auto& input : found.get_target_inputs()) { - remark(13) << input.get_node() << " " << input.get_source_output() << " vs " - << found << " : " << input.get_index() << " " << found.get_index() << std::endl; - if (ov::as_type_ptr(input.get_node()->shared_from_this()) != nullptr && input.get_source_output() == found) { - return input.get_index(); - } + auto input_values = node->input_values(); + /* + * Called with subgraph->input_value(i) arg and used to + * Check that the attached node input subgraph has the same input as the node itself. + * If true, then ternary merge is initiated. + * input + * / \ + * subgraph--node + */ + auto is_recurrent = [&input_values](const ngraph::Output& to_find) -> bool { + return std::any_of(input_values.begin(), input_values.end(), + [&](const ov::Output &in) {return in == to_find;}); + }; + /* + * Checks if the passed node introduces loop dependency for given topological bounds (pair of maxParentOrder, minChildOrder). + * The bounds are presumed to be without dependency. The bounds are updated if no dependency is introduced by the node. + */ + const auto cyclicDependencyIsIntoduced = [&node](const std::shared_ptr& nodeToExamine, std::pair& currentBounds) -> bool { + assert(currentBounds.first < currentBounds.second && "Invalid currentBounds passed"); + const auto &parentNodes = ngraph::as_node_vector(nodeToExamine->input_values()); + const int64_t maxParentOrder = std::accumulate(parentNodes.begin(), parentNodes.end(), currentBounds.first, + [](int64_t maxOrder, std::shared_ptr n){ + if (ngraph::op::is_constant(n) || ngraph::op::is_parameter(n)) + return maxOrder; + return std::max(maxOrder, GetTopologicalOrder(n)); + }); + const auto &childNodes = nodeToExamine->get_users(); + // Skip the node being attached, since it will be a part of subgraph and can't introduce loop dependency + const int64_t minChildOrder = std::accumulate(childNodes.begin(), childNodes.end(), currentBounds.second, + [&node](int64_t minOrder, std::shared_ptr n){ + if (ngraph::op::is_constant(n) || ngraph::op::is_parameter(n) || n == node) + return minOrder; + return std::min(minOrder, GetTopologicalOrder(n)); + }); + if (maxParentOrder < minChildOrder) { + currentBounds = std::pair(maxParentOrder, minChildOrder); + return false; } - return 0; + return true; }; - for (auto input : inputs) { - auto input_node = input.get_source_output().get_node_shared_ptr(); - + for (const auto &input_node : ngraph::as_node_vector(input_values)) { if (auto subgraph = ov::as_type_ptr(input_node)) { if (!clones.count(input_node)) { - auto f = ngraph::clone_function(*subgraph->get_body().get()); + auto f = ov::clone_model(*subgraph->get_body().get()); f->set_friendly_name(subgraph->get_body()->get_friendly_name()); clones[input_node] = f; } } } - - for (auto input : inputs) { - auto input_node = input.get_source_output().get_node_shared_ptr(); - - if (auto subgraph = ov::as_type_ptr(input_node)) { + // If there are no input subgraphs no need to go further, just create a new one. + if (clones.empty()) { + create_single_node_subgraph(node); + remark(1) << "Starting subgraph at: " << node->get_friendly_name() + << " with " << node->inputs().size() << " inputs and " << node->outputs().size() + << " outputs" << std::endl; + return true; + } + std::string newSubgraphName{}; + std::string fusedNames{}; + size_t num_result_children = 0; + std::pair currentTopoBounds {-1, LONG_MAX}; + cyclicDependencyIsIntoduced(node, currentTopoBounds); + assert(!cyclicDependencyIsIntoduced(node, currentTopoBounds) && "Cyclic dependency is introduced by the node itself"); + for (const auto& input_value : input_values) { + auto input_node = input_value.get_node_shared_ptr(); + if (ov::is_type(input_node) && + !cyclicDependencyIsIntoduced(input_node, currentTopoBounds)) { + auto subgraph = std::static_pointer_cast(input_node); if (!input_subgraphs.count(input_node)) { input_subgraphs.insert(input_node); + fusedNames += getFusedNames(subgraph); + if (newSubgraphName.empty()) + newSubgraphName = subgraph->get_friendly_name(); + num_result_children += has_result_child(subgraph); auto f = clones[input_node]; const auto& input_body_parameters = f->get_parameters(); - + // Todo: + // Some of the input subgraphs might have common parents, so some of the input_parameters might already be + // in external_inputs and hence in body_parameters. Here we handle this case and remove repeated body_parameters. + // Would it be better to incorporate all inputs first and then remove repeated params. for (size_t i = 0; i < input_body_parameters.size(); ++i) { auto found = std::find(external_inputs.begin(), external_inputs.end(), subgraph->input_value(i)); if (found != external_inputs.end()) { - auto current_input_index = get_input_index(*found); - // Handling the case if multiple inputs referencing the same parameter comes from one subgraph => it's not introduced by SS. - // It might be better to keep track if body parameter relationship rather than that + // Todo: here we rely on friendly_name uniqueness. Propose a different algorithm. + size_t current_input_index = body_parameters.size(); + for (size_t p_ind = 0; p_ind < body_parameters.size(); p_ind++) { + const auto & p = body_parameters[p_ind]; + if (p->get_friendly_name() == found->get_node_shared_ptr()->get_friendly_name()) { + current_input_index = p_ind; + break; + } + } + if (current_input_index < body_parameters.size()) { remark(13) << "replacing " << *found << " " << current_input_index << " with " << body_parameters[current_input_index] << std::endl; @@ -354,16 +373,19 @@ ngraph::snippets::pass::AttachToSubgraph::AttachToSubgraph(bool tokenize_by_node auto internal = input_body_parameters[i]; auto internal_consumers = internal->outputs(); - for (auto output : internal->outputs()) { - for (auto consumer : output.get_target_inputs()) { - if (auto to_replace_with = ov::as_type_ptr(subgraph->input_value(i).get_node_shared_ptr())) { - auto other_body = clones[subgraph->input_value(i).get_node_shared_ptr()]; - auto other_body_result = other_body->get_results()[consumer.get_source_output().get_index()]; - auto result_producer = other_body_result->input(0).get_source_output(); + if (auto to_replace_with = ov::as_type_ptr(subgraph->get_input_node_shared_ptr(i))) { + for (auto output : internal_consumers) { + for (auto consumer : output.get_target_inputs()) { + auto other_body = clones[subgraph->get_input_node_shared_ptr(i)]; + auto other_body_result = other_body->get_results()[consumer.get_source_output().get_index()]; + auto result_producer = other_body_result->input(0).get_source_output(); - consumer.replace_source_output(result_producer.get_node_shared_ptr()); - } - } + consumer.replace_source_output(result_producer.get_node_shared_ptr()); + } + } + } else { + external_inputs.push_back(subgraph->input_value(i)); + body_parameters.push_back(input_body_parameters[i]); } } else { external_inputs.push_back(subgraph->input_value(i)); @@ -375,7 +397,7 @@ ngraph::snippets::pass::AttachToSubgraph::AttachToSubgraph(bool tokenize_by_node // this is there stitching happens, get result of a copy of a body of currently processed input and put it to the new inputs // internal output index == external output index auto& input_body = clones[input_node]; - size_t source_output_index = input.get_source_output().get_index(); + size_t source_output_index = input_value.get_index(); auto source_result = input_body->get_results()[source_output_index]; // Result op has a single input internal_inputs.push_back(source_result->input_value(0)); @@ -383,15 +405,20 @@ ngraph::snippets::pass::AttachToSubgraph::AttachToSubgraph(bool tokenize_by_node if (op::is_scalar_constant(input_node)) { internal_inputs.push_back(input_node->output(0)); } else { - external_inputs.push_back(input.get_source_output()); - auto new_parameter = std::make_shared(input.get_element_type(), input.get_partial_shape()); - new_parameter->set_friendly_name(input.get_source_output().get_node()->get_friendly_name()); + external_inputs.push_back(input_value); + auto new_parameter = std::make_shared(input_value.get_element_type(), input_value.get_partial_shape()); + new_parameter->set_friendly_name(input_node->get_friendly_name()); body_parameters.push_back(new_parameter); - body_parameters.back()->set_friendly_name(input.get_source_output().get_node()->get_friendly_name()); internal_inputs.push_back(new_parameter->output(0)); } } } + fusedNames += node->get_friendly_name(); + num_result_children += get_num_result_children(node); + if (num_result_children > 1) + return abort_with_strategy("New subgraph is created since too many Result children are detected"); + if (newSubgraphName.empty()) + newSubgraphName = node->get_friendly_name(); auto body_node = node->copy_with_new_inputs(internal_inputs); body_node->set_friendly_name(node->get_friendly_name()); @@ -444,26 +471,20 @@ ngraph::snippets::pass::AttachToSubgraph::AttachToSubgraph(bool tokenize_by_node if (body_results.size() != subgraph_result_inputs.size()) { throw ngraph_error("body results and node results size mismatch during subgraph collaps"); } - + // todo: move this plugin-specific constraint to the plugin callback if (body_parameters.size() + body_results.size() > 7) { - if (strategy == continuation_strategy::reset) { - remark(13) << "new subgraph is created. Impossible to schedule subgraph with " - << body_parameters.size() << " inputs and " << body_results.size() << " outputs." << std::endl; - - auto single_node_subgraph = op::Subgraph::wrap_node_as_subgraph(node); - ngraph::replace_node(node, single_node_subgraph); - return true; - } else { - return false; - } + const std::string message_reset = "new subgraph is created. Impossible to schedule subgraph with " + + std::to_string(body_parameters.size()) + " inputs and " + std::to_string(body_results.size()) + " outputs."; + const std::string message_abort = "failed to continue subgraph. Impossible to schedule subgraph with " + + std::to_string(body_parameters.size()) + " inputs and " + std::to_string(body_results.size()) + " outputs."; + return abort_with_strategy(message_reset, message_abort); } - auto body = op::create_body(node->get_friendly_name(), body_results, body_parameters); + auto body = op::create_body(newSubgraphName, body_results, body_parameters); for (size_t i = 0; i < body->get_parameters().size(); i++) { body->get_parameters()[i]->set_friendly_name(body_parameters[i]->get_friendly_name()); } - - auto subgraph = op::build_subgraph(node, external_inputs, body); + auto subgraph = op::build_subgraph(node, external_inputs, body, newSubgraphName); auto act_body = subgraph->get_body(); for (size_t i = 0; i < act_body->get_parameters().size(); i++) { act_body->get_parameters()[i]->set_friendly_name(body_parameters[i]->get_friendly_name()); @@ -473,37 +494,15 @@ ngraph::snippets::pass::AttachToSubgraph::AttachToSubgraph(bool tokenize_by_node throw ngraph_error("newly create subgraph doesn't much number of results"); } - if (outputs_are_not_broadcastable(subgraph)) { - if (strategy == continuation_strategy::reset) { - remark(13) << "New subgraph is created due to outputs of a subgraph not broadcastable." << std::endl; - - auto single_node_subgraph = op::Subgraph::wrap_node_as_subgraph(node); - single_node_subgraph->validate_and_infer_types(); - ngraph::replace_node(node, single_node_subgraph); - return true; - } else { - return false; - } - } - - if (has_cycles_of_dependencies(subgraph_result_inputs, subgraph->inputs())) { - if (strategy == continuation_strategy::reset) { - remark(13) << "New subgraph is created due to loop dependency introduced by one of input subgraphs." << std::endl; - - auto single_node_subgraph = op::Subgraph::wrap_node_as_subgraph(node); - single_node_subgraph->validate_and_infer_types(); - ngraph::replace_node(node, single_node_subgraph); - return true; - } else { - return false; - } - } + if (outputs_are_not_broadcastable(subgraph)) + return abort_with_strategy("New subgraph is created due to outputs of a subgraph not broadcastable."); for (size_t i = 0; i < subgraph->get_output_size(); ++i) { for (auto target_input : subgraph_result_inputs[i]) { target_input.replace_source_output(subgraph->output(i)); } } + update_out_tensor_name(subgraph); subgraph->validate_and_infer_types(); @@ -511,6 +510,7 @@ ngraph::snippets::pass::AttachToSubgraph::AttachToSubgraph(bool tokenize_by_node for (size_t i = 0; i < act_body1->get_parameters().size(); i++) { act_body1->get_parameters()[i]->set_friendly_name(body_parameters[i]->get_friendly_name()); } + subgraph->get_rt_info()["originalLayersNames"] = node->get_friendly_name(); remark(1) << "Replacement (merge) done for: " << subgraph->get_friendly_name() @@ -520,11 +520,9 @@ ngraph::snippets::pass::AttachToSubgraph::AttachToSubgraph(bool tokenize_by_node return true; }; - - register_matcher(std::make_shared( - std::make_shared(pattern::any_input(), - [](std::shared_ptr n) { - return is_lo(n) && has_supported_in_out(n) && has_subgraph_as_input(n); - })), - continuation_callback); + auto matcher = std::make_shared(label); + register_matcher(matcher, callback); } +} // namespace pass +} // namespace snippets +} // namespace ngraph \ No newline at end of file diff --git a/src/common/snippets/src/pass/insert_load_store.cpp b/src/common/snippets/src/pass/insert_load_store.cpp index c8130a92ae2..abc3bd5b919 100644 --- a/src/common/snippets/src/pass/insert_load_store.cpp +++ b/src/common/snippets/src/pass/insert_load_store.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "itt.hpp" +#include #include "remarks.hpp" #include "snippets/pass/insert_load_store.hpp" @@ -17,6 +17,7 @@ ngraph::snippets::pass::InsertLoad::InsertLoad() { register_matcher(std::make_shared( ngraph::pattern::wrap_type()), [this](ngraph::pattern::Matcher &m) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::InsertLoad") auto root = m.get_match_root(); // check if already has Load as an output @@ -50,6 +51,7 @@ ngraph::snippets::pass::InsertStore::InsertStore() { register_matcher(std::make_shared( ngraph::pattern::wrap_type()), [this](ngraph::pattern::Matcher &m) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::InsertStore") auto root = m.get_match_root(); // check if already has Store as an input diff --git a/src/common/snippets/src/pass/insert_movebroadcast.cpp b/src/common/snippets/src/pass/insert_movebroadcast.cpp index 93640195ab4..727046a54c9 100644 --- a/src/common/snippets/src/pass/insert_movebroadcast.cpp +++ b/src/common/snippets/src/pass/insert_movebroadcast.cpp @@ -3,7 +3,7 @@ // #include "remarks.hpp" -#include "itt.hpp" +#include #include "snippets/pass/insert_movebroadcast.hpp" #include "snippets/snippets_isa.hpp" @@ -138,6 +138,7 @@ auto reset_broacast_config(const std::shared_ptr& op) -> void { ngraph::snippets::pass::InsertMoveBroadcast::InsertMoveBroadcast() { MATCHER_SCOPE(InsertMoveBroadcast); ngraph::graph_rewrite_callback callback = [this](ngraph::pattern::Matcher &m) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::InsertMoveBroadcast") auto root = m.get_match_root(); const auto& values = root->input_values(); if (values.empty()) { @@ -173,8 +174,8 @@ ngraph::snippets::pass::InsertMoveBroadcast::InsertMoveBroadcast() { auto any = std::make_shared(pattern::any_input(), [](std::shared_ptr n) { // should add supports_auto_broadcast to SquaredDifference - return (ngraph::op::supports_auto_broadcast(n) || !!as_type_ptr(n) || !!as_type_ptr(n)) - && n->get_autob().m_type == ngraph::op::AutoBroadcastType::NUMPY; }); + return ((ngraph::op::supports_auto_broadcast(n) || is_type(n) || is_type(n)) && + n->get_autob().m_type == ngraph::op::AutoBroadcastType::NUMPY) || is_type(n); }); register_matcher(std::make_shared(any), callback); -} \ No newline at end of file +} diff --git a/src/common/snippets/src/pass/load_movebroadcast_to_broadcastload.cpp b/src/common/snippets/src/pass/load_movebroadcast_to_broadcastload.cpp index 8fa9a604ddc..bb64309f8b1 100644 --- a/src/common/snippets/src/pass/load_movebroadcast_to_broadcastload.cpp +++ b/src/common/snippets/src/pass/load_movebroadcast_to_broadcastload.cpp @@ -3,7 +3,7 @@ // #include "remarks.hpp" -#include "itt.hpp" +#include #include "snippets/pass/load_movebroadcast_to_broadcastload.hpp" #include "snippets/snippets_isa.hpp" @@ -20,6 +20,7 @@ ngraph::snippets::pass::LoadMoveBroadcastToBroadcastLoad::LoadMoveBroadcastToBro register_matcher(std::make_shared(fbn), [load_pattern, param_pattern](ngraph::pattern::Matcher &m) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::LoadMoveBroadcastToBroadcastLoad") auto root = m.get_match_root(); const auto &pm = m.get_pattern_value_map(); @@ -44,9 +45,9 @@ ngraph::snippets::pass::LoadMoveBroadcastToBroadcastLoad::LoadMoveBroadcastToBro bct[k] = 1; } } - + // Todo: consider refactoring BroadcastLoad, it seems we don't need broadcast_info at this point. broadcastload->set_broadcast_info(bct); - if (broadcastload->is_broadcast(outshape.size()-1)) { + if (inshape.back() == 1 && outshape.back() != 1) { ngraph::copy_runtime_info(root, broadcastload); ngraph::replace_node(root, broadcastload); return true; diff --git a/src/common/snippets/src/pass/vector_to_scalar.cpp b/src/common/snippets/src/pass/vector_to_scalar.cpp index 6bf17fe2f1f..a54186bf488 100644 --- a/src/common/snippets/src/pass/vector_to_scalar.cpp +++ b/src/common/snippets/src/pass/vector_to_scalar.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "itt.hpp" +#include #include "snippets/pass/vector_to_scalar.hpp" #include "snippets/snippets_isa.hpp" @@ -16,6 +16,7 @@ ngraph::snippets::pass::ReplaceLoadsWithScalarLoads::ReplaceLoadsWithScalarLoads register_matcher(std::make_shared( ngraph::pattern::wrap_type()), [this](ngraph::pattern::Matcher &m) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::ReplaceLoadsWithScalarLoads_callback") auto root = m.get_match_root(); auto load = std::make_shared (root->input_value(0)); load->set_friendly_name(root->get_friendly_name()); @@ -30,6 +31,7 @@ ngraph::snippets::pass::ReplaceStoresWithScalarStores::ReplaceStoresWithScalarSt register_matcher(std::make_shared( ngraph::pattern::wrap_type()), [this](ngraph::pattern::Matcher &m) { + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::ReplaceStoresWithScalarStores_callback") auto root = m.get_match_root(); auto store = std::make_shared (root->input_value(0)); store->set_friendly_name(root->get_friendly_name()); diff --git a/src/common/snippets/src/remarks.hpp b/src/common/snippets/src/remarks.hpp index bd40ae83650..3addf90cb7e 100644 --- a/src/common/snippets/src/remarks.hpp +++ b/src/common/snippets/src/remarks.hpp @@ -5,10 +5,10 @@ #pragma once #include - +// Todo: Replace remarks with DEBUG_CAPS class logstreambuf: public std::streambuf { public: - static const int threshold {5}; + static const int threshold {50}; }; template diff --git a/src/tests/functional/inference_engine/skip_tests_config.cpp b/src/tests/functional/inference_engine/skip_tests_config.cpp index 6af60bfd426..bd418b65c19 100644 --- a/src/tests/functional/inference_engine/skip_tests_config.cpp +++ b/src/tests/functional/inference_engine/skip_tests_config.cpp @@ -15,5 +15,9 @@ std::vector disabledTestPatterns() { ".*/NmsLayerTest.*", // Issue 74365: azure is failing after #7351 ".*DetectionOutput8ToDetectionOutput1", + // TODO: Snpippets rewrite this test + ".*TransformationTests\\.DontStartSubgraphSingleOutput.*", + // TODO: Move this test to CPU-specific + ".*TransformationTests\\.DoNotStartSubgraphAfterInputs.*" }; } diff --git a/src/tests/functional/inference_engine/snippets/tokenization.cpp b/src/tests/functional/inference_engine/snippets/tokenization.cpp index b6be642784a..f2a7aaad5c6 100644 --- a/src/tests/functional/inference_engine/snippets/tokenization.cpp +++ b/src/tests/functional/inference_engine/snippets/tokenization.cpp @@ -14,67 +14,187 @@ #include #include "common_test_utils/ngraph_test_utils.hpp" +#include "functional_test_utils/skip_tests_config.hpp" using namespace testing; -using namespace ngraph; +using namespace ov; +using ngraph::snippets::op::Subgraph; +using ngraph::pass::InitNodeInfo; +using ngraph::snippets::pass::EnumerateNodes; +using ngraph::snippets::pass::TokenizeSnippets; +//using ngraph::snippets::pass::CreateSubgraph; -TEST(TransformationTests, StartSubgraphMultipleOutputs) { - std::shared_ptr f(nullptr), f_ref(nullptr); +// Todo: Move this test to CPU-specific +TEST(TransformationTests, DoNotStartSubgraphAfterInputs) { + // Do not start Subgraph after input parameters to avoid U8->FP32 and FP32->U8 conversion pairs + // Todo: Remove this test when U8 support is enabled in SnippetS and StartSubgraph logics is updated + SKIP_IF_CURRENT_TEST_IS_DISABLED() + std::shared_ptr f(nullptr), f_ref(nullptr); { - auto data0 = std::make_shared(element::f32, Shape{2, 3}); - auto data1 = std::make_shared(element::f32, Shape{1, 3}); - auto add = std::make_shared(data0, data1); - auto sub = std::make_shared(add, data1); - auto mul = std::make_shared(add, sub); - f = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); + auto data0 = std::make_shared(element::f32, Shape{2, 3}); + auto data1 = std::make_shared(element::f32, Shape{1, 3}); + const std::vector const_values{3, 2, 10}; + auto const_data = std::make_shared(element::f32, Shape{1, 3}, const_values); + auto add = std::make_shared(data0, data1); + auto sub = std::make_shared(add, const_data); + auto mul = std::make_shared(add, sub); + f = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); pass::Manager m; - m.register_pass(); - m.register_pass(); + m.register_pass(); + // Todo: When moved to CPU-specific tests, uncomment the markup transformation below. + // m.register_pass(); + m.register_pass(); + m.register_pass(); m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); } + ASSERT_EQ(count_ops_of_type(f), 0); +} +TEST(TransformationTests, StartSubgraphMultipleOutputs) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + std::shared_ptr f(nullptr), f_ref(nullptr); { - auto data0 = std::make_shared(element::f32, Shape{2, 3}); - auto data1 = std::make_shared(element::f32, Shape{1, 3}); - auto indata0 = std::make_shared(element::f32, Shape{2, 3}); - auto indata1 = std::make_shared(element::f32, Shape{1, 3}); - auto add = std::make_shared(NodeVector{data0, data1}, - std::make_shared(NodeVector{std::make_shared(indata0, indata1)}, ParameterVector{indata0, indata1})); - auto sub = std::make_shared(add, data1); - auto mul = std::make_shared(add, sub); - f_ref = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); + auto data0 = std::make_shared(element::i32, Shape{2, 3}); + auto data1 = std::make_shared(element::i32, Shape{1, 3}); + auto convert0 = std::make_shared(data0, element::f32); + auto convert1 = std::make_shared(data1, element::f32); + const std::vector const_values{3, 2, 10}; + auto const_data = std::make_shared(element::f32, Shape{1, 3}, const_values); + auto add = std::make_shared(convert0, convert1); + auto sub = std::make_shared(add, const_data); + auto mul = std::make_shared(add, sub); + f = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); + pass::Manager m; + m.register_pass(); + m.register_pass(); + m.register_pass(); + m.run_passes(f); + ASSERT_NO_THROW(check_rt_info(f)); + } + { + auto data0 = std::make_shared(element::i32, Shape{2, 3}); + auto data1 = std::make_shared(element::i32, Shape{1, 3}); + auto convert0 = std::make_shared(data0, element::f32); + auto convert1 = std::make_shared(data1, element::f32); + const std::vector const_values{3, 2, 10}; + auto const_data = std::make_shared(element::f32, Shape{1, 3}, const_values); + auto indata0 = std::make_shared(element::f32, Shape{2, 3}); + auto indata1 = std::make_shared(element::f32, Shape{1, 3}); + auto indata2 = std::make_shared(element::f32, Shape{1, 3}); + auto add = std::make_shared(indata0, indata1); + auto sub = std::make_shared(add, const_data); + auto mul = std::make_shared(NodeVector{convert0, convert1, const_data}, + std::make_shared(NodeVector{std::make_shared(add, sub)}, + ParameterVector{indata0, indata1, indata2})); + f_ref = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); } auto res = compare_functions(f, f_ref); ASSERT_TRUE(res.first) << res.second; } -TEST(TransformationTests, DontStartSubgraphSingleOuptut) { - std::shared_ptr f(nullptr), f_ref(nullptr); +TEST(TransformationTests, TokenizeMulAddSubgraph) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + std::shared_ptr f(nullptr), f_ref(nullptr); { - auto data0 = std::make_shared(element::f32, Shape{2, 3}); - auto data1 = std::make_shared(element::f32, Shape{1, 3}); - auto add = std::make_shared(data0, data1); - auto sub = std::make_shared(add, data1); - auto mul = std::make_shared(data0, sub); - f = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); + auto data_1 = std::make_shared(element::f32, Shape{ 1, 3, 4, 4 }); + auto data_2 = std::make_shared(element::f32, Shape{ 1, 3, 4, 4 }); + auto non_snippet_op = std::make_shared(data_1, data_2); + + auto mul_const_1 = op::v0::Constant::create(element::f32, { 1, 3, 1, 1 }, { 4.f }); + auto mul_1 = std::make_shared(non_snippet_op, mul_const_1); + auto add_const_1 = op::v0::Constant::create(element::f32, { 1, 3, 1, 1 }, { 16.4f }); + auto add_1 = std::make_shared(mul_1, add_const_1); + auto elu = std::make_shared(add_1, 0.01); + + auto mul_const_2 = op::v0::Constant::create(element::f32, { 1, 3, 1, 1 }, { 4.f }); + auto mul_2 = std::make_shared(non_snippet_op, mul_const_2); + auto add_const_2 = op::v0::Constant::create(element::f32, { 1, 3, 1, 1 }, { 16.4f }); + auto add_2 = std::make_shared(mul_2, add_const_2); + auto relu = std::make_shared(add_2); + + auto add = std::make_shared(elu, relu); + auto result = std::make_shared(add); + + f = std::make_shared(ResultVector{ result }, ParameterVector{ data_1, data_2 }); pass::Manager m; - m.register_pass(); - m.register_pass(); + m.register_pass(); + m.register_pass(); + m.register_pass(); + m.run_passes(f); + ASSERT_NO_THROW(check_rt_info(f)); + } + { + auto data_1 = std::make_shared(element::f32, Shape{ 1, 3, 4, 4 }); + auto data_2 = std::make_shared(element::f32, Shape{ 1, 3, 4, 4 }); + + // snippet inputs + auto non_snippet_op = std::make_shared(data_1, data_2); + auto mul_const_1 = op::v0::Constant::create(element::f32, { 1, 3, 1, 1 }, { 4.f }); + auto add_const_1 = op::v0::Constant::create(element::f32, { 1, 3, 1, 1 }, { 16.4f }); + auto mul_const_2 = op::v0::Constant::create(element::f32, { 1, 3, 1, 1 }, { 4.f }); + auto add_const_2 = op::v0::Constant::create(element::f32, { 1, 3, 1, 1 }, { 16.4f }); + + // snippet function + auto snippet_input = std::make_shared(element::f32, Shape{ 1, 3, 4, 4 }); + + auto sn_mul_const_1 = std::make_shared(element::f32, Shape{ 1, 3, 1, 1 }); + auto mul_1 = std::make_shared(snippet_input, sn_mul_const_1); + auto sn_add_const_1 = std::make_shared(element::f32, Shape{ 1, 3, 1, 1 }); + auto add_1 = std::make_shared(mul_1, sn_add_const_1); + auto elu = std::make_shared(add_1, 0.01); + + auto sn_mul_const_2 = std::make_shared(element::f32, Shape{ 1, 3, 1, 1 }); + auto mul_2 = std::make_shared(snippet_input, sn_mul_const_2); + auto sn_add_const_2 = std::make_shared(element::f32, Shape{ 1, 3, 1, 1 }); + auto add_2 = std::make_shared(mul_2, sn_add_const_2); + auto relu = std::make_shared(add_2); + + auto add = std::make_shared(elu, relu); + ParameterVector subgraph_params{ snippet_input, sn_mul_const_1, sn_add_const_1, sn_mul_const_2, sn_add_const_2 }; + auto snippet_function = std::make_shared(NodeVector{ add }, subgraph_params); + + ngraph::NodeVector snippet_inputs{ non_snippet_op, mul_const_1, add_const_1, mul_const_2, add_const_2 }; + auto snippet = std::make_shared(snippet_inputs, snippet_function); + auto result = std::make_shared(snippet); + + f_ref = std::make_shared(NodeVector{ result }, ParameterVector{ data_1, data_2 }); + } + + auto res = compare_functions(f, f_ref); + ASSERT_TRUE(res.first) << res.second; +} + + +TEST(TransformationTests, DontStartSubgraphSingleOutput) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + std::shared_ptr f(nullptr), f_ref(nullptr); + { + auto data0 = std::make_shared(element::f32, Shape{2, 3}); + auto data1 = std::make_shared(element::f32, Shape{1, 3}); + auto add = std::make_shared(data0, data1); + auto sub = std::make_shared(add, data1); + auto mul = std::make_shared(data0, sub); + f = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); + + pass::Manager m; + m.register_pass(); + m.register_pass(); + m.register_pass(); m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); } { - auto data0 = std::make_shared(element::f32, Shape{2, 3}); - auto data1 = std::make_shared(element::f32, Shape{1, 3}); - auto add = std::make_shared(data0, data1); - auto sub = std::make_shared(add, data1); - auto mul = std::make_shared(data0, sub); - f_ref = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); + auto data0 = std::make_shared(element::f32, Shape{2, 3}); + auto data1 = std::make_shared(element::f32, Shape{1, 3}); + auto add = std::make_shared(data0, data1); + auto sub = std::make_shared(add, data1); + auto mul = std::make_shared(data0, sub); + f_ref = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); } auto res = compare_functions(f, f_ref); @@ -82,35 +202,37 @@ TEST(TransformationTests, DontStartSubgraphSingleOuptut) { } TEST(TransformationTests, AttachToSubgraph) { - std::shared_ptr f(nullptr), f_ref(nullptr); + SKIP_IF_CURRENT_TEST_IS_DISABLED() + std::shared_ptr f(nullptr), f_ref(nullptr); { - auto data0 = std::make_shared(element::f32, Shape{2, 3}); - auto data1 = std::make_shared(element::f32, Shape{1, 3}); - auto indata0 = std::make_shared(element::f32, Shape{2, 3}); - auto indata1 = std::make_shared(element::f32, Shape{1, 3}); - auto add = std::make_shared(NodeVector{data0, data1}, - std::make_shared(NodeVector{std::make_shared(indata0, indata1)}, ParameterVector{indata0, indata1})); - auto neg = std::make_shared(add); - auto concat = std::make_shared(NodeVector{add, neg}, 0); - f = std::make_shared(NodeVector{concat}, ParameterVector{data0, data1}); + auto data0 = std::make_shared(element::f32, Shape{2, 3}); + auto data1 = std::make_shared(element::f32, Shape{1, 3}); + auto indata0 = std::make_shared(element::f32, Shape{2, 3}); + auto indata1 = std::make_shared(element::f32, Shape{1, 3}); + auto add = std::make_shared(NodeVector{data0, data1}, + std::make_shared(NodeVector{std::make_shared(indata0, indata1)}, ParameterVector{indata0, indata1})); + auto neg = std::make_shared(add); + auto concat = std::make_shared(NodeVector{add, neg}, 0); + f = std::make_shared(NodeVector{concat}, ParameterVector{data0, data1}); pass::Manager m; - m.register_pass(); - m.register_pass(); + m.register_pass(); + m.register_pass(); + m.register_pass(); m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); } { - auto data0 = std::make_shared(element::f32, Shape{2, 3}); - auto data1 = std::make_shared(element::f32, Shape{1, 3}); - auto indata0 = std::make_shared(element::f32, Shape{2, 3}); - auto indata1 = std::make_shared(element::f32, Shape{1, 3}); - auto inner = std::make_shared(indata0, indata1); - auto add = std::make_shared(NodeVector{data0, data1}, - std::make_shared(NodeVector{std::make_shared(inner), inner}, ParameterVector{indata0, indata1})); - auto concat = std::make_shared(OutputVector{add->output(0), add->output(1)}, 0); - f_ref = std::make_shared(NodeVector{concat}, ParameterVector{data0, data1}); + auto data0 = std::make_shared(element::f32, Shape{2, 3}); + auto data1 = std::make_shared(element::f32, Shape{1, 3}); + auto indata0 = std::make_shared(element::f32, Shape{2, 3}); + auto indata1 = std::make_shared(element::f32, Shape{1, 3}); + auto inner = std::make_shared(indata0, indata1); + auto add = std::make_shared(NodeVector{data0, data1}, + std::make_shared(NodeVector{std::make_shared(inner), inner}, ParameterVector{indata0, indata1})); + auto concat = std::make_shared(OutputVector{add->output(0), add->output(1)}, 0); + f_ref = std::make_shared(NodeVector{concat}, ParameterVector{data0, data1}); } auto res = compare_functions(f, f_ref); @@ -118,37 +240,48 @@ TEST(TransformationTests, AttachToSubgraph) { } TEST(TransformationTests, DontAttachToSubgraphIfLoop) { - std::shared_ptr f(nullptr), f_ref(nullptr); + SKIP_IF_CURRENT_TEST_IS_DISABLED() + std::shared_ptr f(nullptr), f_ref(nullptr); { - auto data0 = std::make_shared(element::f32, Shape{2, 3}); - auto data1 = std::make_shared(element::f32, Shape{1, 3}); - auto indata0 = std::make_shared(element::f32, Shape{2, 3}); - auto indata1 = std::make_shared(element::f32, Shape{1, 3}); - auto add = std::make_shared(NodeVector{data0, data1}, - std::make_shared(NodeVector{std::make_shared(indata0, indata1)}, ParameterVector{indata0, indata1})); - auto log = std::make_shared(add); - auto mul = std::make_shared(add, log); - f = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); + auto data0 = std::make_shared(element::f32, Shape{2, 3}); + auto data1 = std::make_shared(element::f32, Shape{1, 3}); + auto indata0 = std::make_shared(element::f32, Shape{2, 3}); + auto indata1 = std::make_shared(element::f32, Shape{1, 3}); + auto add = std::make_shared(NodeVector{data0, data1}, + std::make_shared(NodeVector{std::make_shared(indata0, indata1)}, ParameterVector{indata0, indata1})); + auto log = std::make_shared(add); + auto mul = std::make_shared(add, log); + f = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); pass::Manager m; - m.register_pass(); - m.register_pass(); + m.register_pass(); + m.register_pass(); + m.register_pass(); m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); } { - auto data0 = std::make_shared(element::f32, Shape{2, 3}); - auto data1 = std::make_shared(element::f32, Shape{1, 3}); - auto indata0 = std::make_shared(element::f32, Shape{2, 3}); - auto indata1 = std::make_shared(element::f32, Shape{1, 3}); - auto add = std::make_shared(NodeVector{data0, data1}, - std::make_shared(NodeVector{std::make_shared(indata0, indata1)}, ParameterVector{indata0, indata1})); - auto log = std::make_shared(add); - auto mul = std::make_shared(add, log); - f_ref = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); + auto data0 = std::make_shared(element::f32, Shape{2, 3}); + auto data1 = std::make_shared(element::f32, Shape{1, 3}); + auto indata0 = std::make_shared(element::f32, Shape{2, 3}); + auto indata1 = std::make_shared(element::f32, Shape{1, 3}); + auto add = std::make_shared(NodeVector{data0, data1}, + std::make_shared(NodeVector{std::make_shared(indata0, indata1)}, ParameterVector{indata0, indata1})); + auto log = std::make_shared(add); + /* + * Note that log is not currently supported by snippets, so it won't be converted to subgraph. + * Mul will be converted for the "reset" continuation strategy, (present case) + * or left as-is for the "abort" continuation strategy + */ + auto add_param = std::make_shared(element::f32, add->get_output_shape(0)); + auto log_param = std::make_shared(element::f32, log->get_output_shape(0)); + auto mul = std::make_shared(NodeVector{add, log}, + std::make_shared(NodeVector{std::make_shared(add_param, log_param)}, + ParameterVector{add_param, log_param})); + f_ref = std::make_shared(NodeVector{mul}, ParameterVector{data0, data1}); } auto res = compare_functions(f, f_ref); ASSERT_TRUE(res.first) << res.second; -} \ No newline at end of file +} diff --git a/src/tests/unit/cpu/CMakeLists.txt b/src/tests/unit/cpu/CMakeLists.txt index d44d2584212..7f66c147d95 100644 --- a/src/tests/unit/cpu/CMakeLists.txt +++ b/src/tests/unit/cpu/CMakeLists.txt @@ -23,6 +23,8 @@ addIeTargetTest( ov_shape_inference inference_engine_s unitTestUtils + inference_engine_snippets + ngraphFunctions ADD_CPPLINT LABELS CPU @@ -31,3 +33,8 @@ addIeTargetTest( ie_faster_build(${TARGET_NAME} UNITY ) + +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE + ${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/src/common + ${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/src/cpu + ${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/include) diff --git a/src/tests/unit/cpu/generate_add.cpp b/src/tests/unit/cpu/generate_add.cpp new file mode 100644 index 00000000000..a1069d58957 --- /dev/null +++ b/src/tests/unit/cpu/generate_add.cpp @@ -0,0 +1,427 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include +#include + +#include + +#include +#include + +#include + +using namespace testing; + +auto gen_inputs(const ngraph::Shape& shape, size_t n = 2) -> std::vector>{ + std::vector> referenceInputs(n); + for (size_t k = 0; k < n; k++) { + referenceInputs[k].resize(ngraph::shape_size(shape)*sizeof(float)); + float* in0 = reinterpret_cast(&referenceInputs[k][0]); + + for (size_t i = 0; i < ngraph::shape_size(shape); i++) { + if (k % 3 == 0) { + in0[i] = i / 2048.f; + } else if (k % 3 == 1) { + in0[i] = 1 - i / 2048.f; + } else { + in0[i] = i / 1024.f; + } + } + } + return referenceInputs; +} + +auto compare(std::shared_ptr& s, std::shared_ptr& f, std::vector>& in) -> bool{ + auto act = ngraph::helpers::interpreterFunction(s, in); + auto exp = ngraph::helpers::interpreterFunction(f, in); + + const float* pexp = reinterpret_cast(&exp[0].second[0]); + const float* pact = reinterpret_cast(&act[0].second[0]); + + bool isCorrect = true; + for (int i = 0; i < ngraph::shape_size(f->get_result()->get_shape()); i++) { + if (std::abs(pexp[i]-pact[i]) > std::numeric_limits::epsilon() + || std::isnan(pexp[i]) != std::isnan(pact[i])) { + isCorrect = false; + std::cout << i << " expected " << pexp[i] << " actual " << pact[i] << " diff " << std::abs(pexp[i]-pact[i]) << std::endl; + } + } + return isCorrect; +} + +auto wrapAsSnippet(std::shared_ptr& f, const ngraph::Shape& shape0, const ngraph::Shape& shape1) +-> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape0); + auto input1 = std::make_shared(ngraph::element::f32, shape1); + auto snippet = std::make_shared(ngraph::OutputVector{input0, input1}, ngraph::clone_function(*f.get())); + return std::make_shared(ngraph::NodeVector{snippet}, ngraph::ParameterVector{input0, input1}); +} + +auto wrapAsSnippet(std::shared_ptr& f, const ngraph::Shape& shape0) +-> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape0); + auto snippet = std::make_shared(ngraph::OutputVector{input0}, ngraph::clone_function(*f.get())); + return std::make_shared(ngraph::NodeVector{snippet}, ngraph::ParameterVector{input0}); +} + +// Todo: Reimplement Snippets tests, so they won't require evaluate method for OP Subgraph +// In more detail, we can't use interpreter backend for op::Subgraph evaluation, since it will +// depend on the snippets lib in this (which is not allowed). +TEST(SnippetsTests, GenerateAddParams) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + + auto f = ([] (const ngraph::Shape& shape) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape); + auto input1 = std::make_shared(ngraph::element::f32, shape); + auto add = std::make_shared(input0, input1); + + return std::make_shared(ngraph::NodeVector{add}, ngraph::ParameterVector{input0, input1}); + })(shape); + + auto s = wrapAsSnippet(f, shape, shape); + auto referenceInputs = gen_inputs(shape, 2); + bool isCorrect = compare(s, f, referenceInputs); + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddConstant) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + + auto f = ([] (const ngraph::Shape& shape) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape); + + std::vector vals(ngraph::shape_size(shape)); + for (int i = 0; i < ngraph::shape_size(shape); i++) { + vals[i] = 1-i/2048.f; + } + auto input1 = std::make_shared(ngraph::element::f32, shape, vals); + auto add = std::make_shared(input0, input1); + + return std::make_shared(ngraph::NodeVector{add}, ngraph::ParameterVector{input0}); + })(shape); + + auto s = wrapAsSnippet(f, shape); + auto referenceInputs = gen_inputs(shape, 1); + bool isCorrect = compare(s, f, referenceInputs); + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddConstantScalar) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + + auto f = ([] (const ngraph::Shape& shape) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape); + auto input1 = std::make_shared(ngraph::element::f32, ngraph::Shape{1}, std::vector({42.f})); + auto add = std::make_shared(input0, input1); + + return std::make_shared(ngraph::NodeVector{add}, ngraph::ParameterVector{input0}); + })(shape); + + auto s = wrapAsSnippet(f, shape); + auto referenceInputs = gen_inputs(shape, 1); + bool isCorrect = compare(s, f, referenceInputs); + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddConstantScalarEmptySize) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + + auto f = ([] (const ngraph::Shape& shape) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape); + auto input1 = std::make_shared(ngraph::element::f32, ngraph::Shape(), std::vector({42.f})); + auto add = std::make_shared(input0, input1); + + return std::make_shared(ngraph::NodeVector{add}, ngraph::ParameterVector{input0}); + })(shape); + + auto s = wrapAsSnippet(f, shape); + auto referenceInputs = gen_inputs(shape, 1); + bool isCorrect = compare(s, f, referenceInputs); + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddBroadcastX2Edges) { + GTEST_SKIP(); + auto shape0 = ngraph::Shape{1, 4, 16, 31}; + auto shape1 = ngraph::Shape{1, 4, 16, 1}; + + auto f = ([] (const ngraph::Shape& shape0, const ngraph::Shape& shape1) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape0); + auto input1 = std::make_shared(ngraph::element::f32, shape1); + auto input2 = std::make_shared(ngraph::element::f32, shape1); + + auto add = std::make_shared(input0, input1); + auto mul = std::make_shared(input1, input2); + + auto sub = std::make_shared(add, mul); + + return std::make_shared(ngraph::NodeVector{sub}, ngraph::ParameterVector{input0, input1, input2}); + })(shape0, shape1); + + auto s = ([f] (const ngraph::Shape& shape0, const ngraph::Shape& shape1) -> std::shared_ptr{ + auto input2 = std::make_shared(ngraph::element::f32, shape0); + auto input3 = std::make_shared(ngraph::element::f32, shape1); + auto input4 = std::make_shared(ngraph::element::f32, shape1); + auto snippet = std::make_shared(ngraph::OutputVector{input2, input3, input4}, ngraph::clone_function(*f.get())); + return std::make_shared(ngraph::NodeVector{snippet}, ngraph::ParameterVector{input2, input3, input4}); + })(shape0, shape1); + + std::vector> referenceInputs(3); + referenceInputs[0].resize(ngraph::shape_size(shape0)*sizeof(float)); + referenceInputs[1].resize(ngraph::shape_size(shape1)*sizeof(float)); + referenceInputs[2].resize(ngraph::shape_size(shape1)*sizeof(float)); + + float* in0 = reinterpret_cast(&referenceInputs[0][0]); + float* in1 = reinterpret_cast(&referenceInputs[1][0]); + float* in2 = reinterpret_cast(&referenceInputs[2][0]); + for (int i = 0; i < ngraph::shape_size(shape0); i++) { + in0[i] = i/2048.f; + } + + in1[0] = 1.f; + in2[0] = 0.42f; + + auto act = ngraph::helpers::interpreterFunction(s, referenceInputs); + auto exp = ngraph::helpers::interpreterFunction(f, referenceInputs); + + const float* pexp = reinterpret_cast(&exp[0].second[0]); + const float* pact = reinterpret_cast(&act[0].second[0]); + + bool isCorrect = true; + for (int i = 0; i < ngraph::shape_size(shape0); i++) { + if (pexp[i] != pact[i]) { + isCorrect = false; + std::cout << i << " expected " << pexp[i] << " actual " << pact[i] << std::endl; + } + } + + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddBroadcastY) { + GTEST_SKIP(); + auto shape0 = ngraph::Shape{1, 4, 16, 31}; + auto shape1 = ngraph::Shape{1, 4, 1, 31}; + + auto f = ([] (const ngraph::Shape& shape0, const ngraph::Shape& shape1) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape0); + auto input1 = std::make_shared(ngraph::element::f32, shape1); + auto add = std::make_shared(input0, input1); + + return std::make_shared(ngraph::NodeVector{add}, ngraph::ParameterVector{input0, input1}); + })(shape0, shape1); + + auto s = wrapAsSnippet(f, shape0, shape1); + + std::vector> referenceInputs(2); + referenceInputs[0].resize(ngraph::shape_size(shape0)*sizeof(float)); + referenceInputs[1].resize(ngraph::shape_size(shape1)*sizeof(float)); + + float* in0 = reinterpret_cast(&referenceInputs[0][0]); + float* in1 = reinterpret_cast(&referenceInputs[1][0]); + for (size_t i = 0; i < ngraph::shape_size(shape0); i++) { + in0[i] = i / 2048.f; + } + for (size_t i = 0; i < ngraph::shape_size(shape1); i++) { + in1[i] = 1 - i / 2048.f; + } + + auto act = ngraph::helpers::interpreterFunction(s, referenceInputs); + auto exp = ngraph::helpers::interpreterFunction(f, referenceInputs); + + const float* pexp = reinterpret_cast(&exp[0].second[0]); + const float* pact = reinterpret_cast(&act[0].second[0]); + + bool isCorrect = true; + for (size_t i = 0; i < ngraph::shape_size(shape0); i++) { + if (pexp[i] != pact[i]) { + isCorrect = false; + std::cout << i << " expected " << pexp[i] << " actual " << pact[i] << std::endl; + } + } + + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddNegate) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + + auto f = ([] (const ngraph::Shape& shape) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape); + auto input1 = std::make_shared(ngraph::element::f32, shape); + auto add = std::make_shared(input0, input1); + auto nagate = std::make_shared(add); + + return std::make_shared(ngraph::NodeVector{nagate}, ngraph::ParameterVector{input0, input1}); + })(shape); + + auto s = ([f] (const ngraph::Shape& shape) -> std::shared_ptr{ + auto input2 = std::make_shared(ngraph::element::f32, shape); + auto input3 = std::make_shared(ngraph::element::f32, shape); + auto snippet = std::make_shared(ngraph::OutputVector{input2, input3}, ngraph::clone_function(*f.get())); + return std::make_shared(ngraph::NodeVector{snippet}, ngraph::ParameterVector{input2, input3}); + })(shape); + + auto referenceInputs = gen_inputs(shape, 2); + bool isCorrect = compare(s, f, referenceInputs); + + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddNegateAdd) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + auto input1 = std::make_shared(ngraph::element::f32, shape); + auto input2 = std::make_shared(ngraph::element::f32, shape); + auto add = std::make_shared(input1, input2); + auto nagate = std::make_shared(add); + auto input3 = std::make_shared(ngraph::element::f32, shape); + auto add2 = std::make_shared(nagate, input3); + std::shared_ptr f = std::make_shared(ngraph::NodeVector{add2}, ngraph::ParameterVector{input1, input2, input3}); + + auto input11 = std::make_shared(ngraph::element::f32, shape); + auto input21 = std::make_shared(ngraph::element::f32, shape); + auto input31 = std::make_shared(ngraph::element::f32, shape); + auto snippet = std::make_shared(ngraph::OutputVector{input11, input21, input31}, ngraph::clone_function(*f.get())); + std::shared_ptr s = std::make_shared(ngraph::NodeVector{snippet}, ngraph::ParameterVector{input11, input21, input31}); + + auto referenceInputs = gen_inputs(shape, 3); + bool isCorrect = compare(s, f, referenceInputs); + + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddNegateAddMultiEdge) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + auto input1 = std::make_shared(ngraph::element::f32, shape); + auto input2 = std::make_shared(ngraph::element::f32, shape); + auto add = std::make_shared(input1, input2); + auto nagate = std::make_shared(add); + auto add2 = std::make_shared(nagate, input1); + std::shared_ptr f = std::make_shared(ngraph::NodeVector{add2}, ngraph::ParameterVector{input1, input2}); + + auto s = wrapAsSnippet(f, shape, shape); + auto referenceInputs = gen_inputs(shape, 2); + bool isCorrect = compare(s, f, referenceInputs); + + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateAddNegateAddMultiEdgeConst) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + auto input1 = std::make_shared(ngraph::element::f32, shape); + auto input2 = ngraph::op::Constant::create(ngraph::element::f32, ngraph::Shape{}, {0.42}); + auto add = std::make_shared(input1, input2); + auto nagate = std::make_shared(add); + auto add2 = std::make_shared(nagate, input1); + std::shared_ptr f = std::make_shared(ngraph::NodeVector{add2}, ngraph::ParameterVector{input1}); + + auto input11 = std::make_shared(ngraph::element::f32, shape); + auto snippet = std::make_shared(ngraph::OutputVector{input11}, ngraph::clone_function(*f.get())); + std::shared_ptr s = std::make_shared(ngraph::NodeVector{snippet}, ngraph::ParameterVector{input11}); + + auto referenceInputs = gen_inputs(shape, 1); + bool isCorrect = compare(s, f, referenceInputs); + + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +TEST(SnippetsTests, GenerateErf) { + GTEST_SKIP(); + auto shape = ngraph::Shape{1, 4, 16, 31}; + + auto input1 = std::make_shared(ngraph::element::f32, shape); + auto gelu = std::make_shared(input1); + std::shared_ptr f = std::make_shared(ngraph::NodeVector{gelu}, ngraph::ParameterVector{input1}); + + auto input11 = std::make_shared(ngraph::element::f32, shape); + auto snippet = std::make_shared(ngraph::OutputVector{input11}, ngraph::clone_function(*f.get())); + std::shared_ptr s = std::make_shared(ngraph::NodeVector{snippet}, ngraph::ParameterVector{input11}); + + auto referenceInputs = gen_inputs(shape, 1); + bool isCorrect = compare(s, f, referenceInputs); + + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +} + +// ToDO: implement tile selection logic & broadcast emission to make it working. Broadcast substitution works +TEST(SnippetsTests, GenerateAddBroadcastAutomatic) { + GTEST_SKIP(); + std::array shapes { + ngraph::Shape{1, 4, 16, 31}, + ngraph::Shape{1, 4, 16, 1}, + ngraph::Shape{1, 4, 16, 1} + }; + + auto f = ([] (const ngraph::Shape& shape0, const ngraph::Shape& shape1, const ngraph::Shape& shape2) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape0); + auto input1 = std::make_shared(ngraph::element::f32, shape1); + auto input2 = std::make_shared(ngraph::element::f32, shape2); + + auto add = std::make_shared(input0, input1); + auto mul = std::make_shared(input1, input2); + auto sub = std::make_shared(add, mul); + + return std::make_shared(ngraph::NodeVector{sub}, ngraph::ParameterVector{input0, input1, input2}); + })(shapes[0], shapes[1], shapes[2]); + + auto s = ([f] (const ngraph::Shape& shape0, const ngraph::Shape& shape1, const ngraph::Shape& shape2) -> std::shared_ptr{ + auto input0 = std::make_shared(ngraph::element::f32, shape0); + auto input1 = std::make_shared(ngraph::element::f32, shape1); + auto input2 = std::make_shared(ngraph::element::f32, shape2); + auto snippet = std::make_shared(ngraph::OutputVector{input0, input1, input2}, ngraph::clone_function(*f.get())); + return std::make_shared(ngraph::NodeVector{snippet}, ngraph::ParameterVector{input0, input1, input2}); + })(shapes[0], shapes[1], shapes[2]); + + std::vector> referenceInputs(3); + for (int k = 0; k < referenceInputs.size(); k++) { + referenceInputs[k].resize(ngraph::shape_size(shapes[k]) * sizeof(float)); + + auto in0 = reinterpret_cast(&referenceInputs[k][0]); + for (int i = 0; i < ngraph::shape_size(shapes[k]); i++) { + in0[i] = k == 0 ? i/2048.f : (k == 1 ? 1.f : 0.42f); + } + } + + auto exp = ngraph::helpers::interpreterFunction(f, referenceInputs); + auto act = ngraph::helpers::interpreterFunction(s, referenceInputs); + + const float* pexp = reinterpret_cast(&exp[0].second[0]); + const float* pact = reinterpret_cast(&act[0].second[0]); + + bool isCorrect = true; + for (size_t i = 0; i < ngraph::shape_size(shapes[0]); i++) { + if (pexp[i] != pact[i]) { + isCorrect = false; + std::cout << i << " expected " << pexp[i] << " actual " << pact[i] << std::endl; + } + } + + ASSERT_TRUE(isCorrect) << "snippet and native implementation differs"; +}