From e534efd4a893d4c8df5962c0dbcd2d21020100ee Mon Sep 17 00:00:00 2001 From: Ilya Churaev Date: Wed, 1 Mar 2023 07:12:33 +0400 Subject: [PATCH] Moved template backend to new API (#15878) * Moved template backend to new API * Fixed compilation * Fixed some comments * Fixed ov_core_unit_tests * Fixed some tests * Fixed ONNX Frontend tests * Fixed transformation tests * Fixed dynamic tests * Fixed sporadic in CPU tests * Added WA for plugin * Fixed copy_to for scalar tensors --- src/core/src/node.cpp | 6 +- src/core/src/runtime/ov_tensor.cpp | 12 +- src/core/tests/ov_tensor_test.cpp | 20 +- src/plugins/template/backend/CMakeLists.txt | 11 +- src/plugins/template/backend/backend.cpp | 31 +- src/plugins/template/backend/backend.hpp | 72 +---- src/plugins/template/backend/cache.cpp | 95 ------ src/plugins/template/backend/cache.hpp | 45 --- .../template/backend/evaluates_map.cpp | 44 +-- src/plugins/template/backend/executable.cpp | 120 ++----- src/plugins/template/backend/executable.hpp | 104 ++---- src/plugins/template/backend/int_backend.cpp | 53 +--- src/plugins/template/backend/int_backend.hpp | 29 +- .../template/backend/int_executable.cpp | 299 +++++++++--------- .../template/backend/int_executable.hpp | 61 ++-- .../template/backend/pass/dyn_elimination.cpp | 128 -------- .../template/backend/pass/dyn_elimination.hpp | 47 --- .../template/backend/pass/shape_relevance.cpp | 98 ------ .../template/backend/pass/shape_relevance.hpp | 17 - .../template/backend/performance_counter.hpp | 37 --- src/plugins/template/src/plugin.cpp | 2 +- src/plugins/template/src/plugin.hpp | 2 +- .../template/src/sync_infer_request.cpp | 23 +- .../template/src/sync_infer_request.hpp | 6 +- .../src/utils/ngraph_helpers.cpp | 40 ++- 25 files changed, 331 insertions(+), 1071 deletions(-) delete mode 100644 src/plugins/template/backend/cache.cpp delete mode 100644 src/plugins/template/backend/cache.hpp delete mode 100644 src/plugins/template/backend/pass/dyn_elimination.cpp delete mode 100644 src/plugins/template/backend/pass/dyn_elimination.hpp delete mode 100644 src/plugins/template/backend/pass/shape_relevance.cpp delete mode 100644 src/plugins/template/backend/pass/shape_relevance.hpp delete mode 100644 src/plugins/template/backend/performance_counter.hpp diff --git a/src/core/src/node.cpp b/src/core/src/node.cpp index 0fe01e34afe..20713b7ab8a 100644 --- a/src/core/src/node.cpp +++ b/src/core/src/node.cpp @@ -730,7 +730,8 @@ bool ov::Node::evaluate(ov::TensorVector& output_values, const ov::TensorVector& OPENVINO_SUPPRESS_DEPRECATED_START bool sts = evaluate(output, input); OPENVINO_SUPPRESS_DEPRECATED_END - update_output_tensors(output_values, output); + if (sts) + update_output_tensors(output_values, output); return sts; } @@ -743,7 +744,8 @@ bool ov::Node::evaluate(ov::TensorVector& output_values, OPENVINO_SUPPRESS_DEPRECATED_START bool sts = evaluate(output, input, evaluationContext); OPENVINO_SUPPRESS_DEPRECATED_END - update_output_tensors(output_values, output); + if (sts) + update_output_tensors(output_values, output); // Call evaluate for ov::Tensor if op doesn't have evaluate with EvaluationContext return sts ? sts : evaluate(output_values, input_values); } diff --git a/src/core/src/runtime/ov_tensor.cpp b/src/core/src/runtime/ov_tensor.cpp index e4d46691d19..f57cda93ade 100644 --- a/src/core/src/runtime/ov_tensor.cpp +++ b/src/core/src/runtime/ov_tensor.cpp @@ -128,6 +128,13 @@ Shape Tensor::get_shape() const { } void Tensor::copy_to(ov::Tensor& dst) const { + const auto& is_scalar = [](const ov::Shape& shape) { + return shape.empty() || (shape.size() == 1 && shape[0] == 1); + }; + const auto shapes_equal = [is_scalar](const ov::Shape& src, const ov::Shape& dst) { + // WA for scalar tensors to copy {1} to {} or otherwise + return src == dst || (is_scalar(src) && is_scalar(dst)); + }; OV_TENSOR_STATEMENT({ OPENVINO_ASSERT(dst, "Destination tensor was not initialized."); OPENVINO_ASSERT(!is(), "Default copy to doesn't support copy from remote tensor."); @@ -140,7 +147,7 @@ void Tensor::copy_to(ov::Tensor& dst) const { ")"); if (dst.get_shape() == ov::Shape{0}) dst.set_shape(get_shape()); - OPENVINO_ASSERT(dst.get_shape() == get_shape(), + OPENVINO_ASSERT(shapes_equal(get_shape(), dst.get_shape()), "Tensor shapes are not equal. (src: ", get_shape(), " != dst: ", @@ -154,7 +161,8 @@ void Tensor::copy_to(ov::Tensor& dst) const { ov::Shape cur_pos{0}; ov::Shape max_pos{1}; - if (get_element_type().bitwidth() < 8 || (get_strides() == dst.get_strides() && is_continuous())) { + if (get_element_type().bitwidth() < 8 || (get_strides() == dst.get_strides() && is_continuous()) || + (is_scalar(get_shape()) && is_scalar(dst.get_shape()))) { // OpenVINO doesn't support strides for LP types // or both tensors have default strides // Strides and positions already initialized diff --git a/src/core/tests/ov_tensor_test.cpp b/src/core/tests/ov_tensor_test.cpp index 98dbf9f2383..86f91bb63da 100644 --- a/src/core/tests/ov_tensor_test.cpp +++ b/src/core/tests/ov_tensor_test.cpp @@ -323,8 +323,10 @@ std::vector fill_data(const ov::Tensor& tensor) { const T* data = tensor.data(); auto strides = tensor.get_strides(); for (auto&& c : ngraph::CoordinateTransformBasic{tensor.get_shape()}) { - actual.emplace_back( - *(data + (c[2] * strides[2] + c[1] * strides[1] + c[0] * strides[0]) / tensor.get_element_type().size())); + size_t offset = 0; + for (size_t i = 0; i < strides.size(); i++) + offset += c[i] * strides[i]; + actual.emplace_back(*(data + offset / tensor.get_element_type().size())); } return actual; }; @@ -394,7 +396,7 @@ void init_tensor(const ov::Tensor& tensor, bool input) { void compare_tensors(const ov::Tensor& src, const ov::Tensor& dst) { ASSERT_EQ(src.get_byte_size(), dst.get_byte_size()); - ASSERT_EQ(src.get_shape(), dst.get_shape()); + ASSERT_EQ(src.get_size(), dst.get_size()); ASSERT_EQ(src.get_element_type(), dst.get_element_type()); switch (src.get_element_type()) { case ov::element::bf16: @@ -501,6 +503,18 @@ INSTANTIATE_TEST_SUITE_P(copy_tests, TestParams { ov::Shape{3, 2, 2}, ov::Strides{64, 16, 8}, ov::Shape{3, 2, 2}, ov::Strides{128, 24, 8} + }, + TestParams { + ov::Shape{}, {}, + {}, {} + }, + TestParams { + ov::Shape{1}, {}, + {}, {} + }, + TestParams { + ov::Shape{}, {}, + {1}, {} } ))); // clang-format on diff --git a/src/plugins/template/backend/CMakeLists.txt b/src/plugins/template/backend/CMakeLists.txt index 04b16c14885..16645f0df8f 100644 --- a/src/plugins/template/backend/CMakeLists.txt +++ b/src/plugins/template/backend/CMakeLists.txt @@ -5,16 +5,11 @@ set (SRC backend.cpp backend.hpp - cache.cpp - cache.hpp executable.cpp executable.hpp - performance_counter.hpp - pass/dyn_elimination.cpp - pass/dyn_elimination.hpp - pass/shape_relevance.cpp - pass/shape_relevance.hpp - int_backend.cpp int_executable.cpp evaluates_map.cpp + int_backend.cpp + int_executable.cpp + evaluates_map.cpp ) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") diff --git a/src/plugins/template/backend/backend.cpp b/src/plugins/template/backend/backend.cpp index f4d7bfbbf46..7605b8b5fb1 100644 --- a/src/plugins/template/backend/backend.cpp +++ b/src/plugins/template/backend/backend.cpp @@ -9,36 +9,13 @@ #include "int_backend.hpp" #include "ngraph/file_util.hpp" #include "ngraph/util.hpp" +#include "openvino/core/except.hpp" #include "openvino/util/file_util.hpp" -using namespace std; -using namespace ngraph; +ov::runtime::Backend::~Backend() = default; -runtime::Backend::~Backend() {} - -std::shared_ptr runtime::Backend::create() { - auto inner_backend = make_shared(); +std::shared_ptr ov::runtime::Backend::create() { + auto inner_backend = std::make_shared(); return inner_backend; } - -std::shared_ptr runtime::Backend::create_dynamic_tensor( - const ngraph::element::Type& /* element_type */, - const PartialShape& /* shape */) { - throw std::invalid_argument("This backend does not support dynamic tensors"); -} - -bool runtime::Backend::is_supported(const Node& /* node */) const { - // The default behavior is that a backend does not support any ops. If this is not the case - // then override this method and enhance. - return false; -} - -std::shared_ptr runtime::Backend::load(istream& /* input_stream */) { - throw runtime_error("load operation unimplemented."); -} - -bool runtime::Backend::set_config(const map& /* config */, string& error) { - error = "set_config not supported"; - return false; -} diff --git a/src/plugins/template/backend/backend.hpp b/src/plugins/template/backend/backend.hpp index db44b890726..22c440c55ad 100644 --- a/src/plugins/template/backend/backend.hpp +++ b/src/plugins/template/backend/backend.hpp @@ -8,23 +8,16 @@ #include #include "executable.hpp" -#include "ngraph/function.hpp" -#include "ngraph/shape.hpp" -#include "ngraph/type/element_type.hpp" -#include "ngraph/util.hpp" -#include "performance_counter.hpp" +#include "openvino/core/model.hpp" +#include "openvino/core/type/element_type.hpp" -namespace ngraph { +namespace ov { namespace runtime { -class Tensor; -class Backend; -} // namespace runtime -} // namespace ngraph /// \brief Interface to a generic backend. /// /// Backends are responsible for function execution and value allocation. -class ngraph::runtime::Backend { +class Backend { public: virtual ~Backend(); /// \brief Create a new Backend object @@ -44,14 +37,13 @@ public: /// passed as an output to a function the tensor will have a type and shape after executing /// a call. /// \returns shared_ptr to a new backend-specific tensor - virtual std::shared_ptr create_tensor() = 0; + virtual ov::Tensor create_tensor() = 0; /// \brief Create a tensor specific to this backend /// \param element_type The type of the tensor element /// \param shape The shape of the tensor /// \returns shared_ptr to a new backend-specific tensor - virtual std::shared_ptr create_tensor(const ngraph::element::Type& element_type, - const Shape& shape) = 0; + virtual ov::Tensor create_tensor(const ov::element::Type& element_type, const Shape& shape) = 0; /// \brief Create a tensor specific to this backend /// \param element_type The type of the tensor element @@ -60,59 +52,23 @@ public: /// must be sufficient to contain the tensor. The lifetime of the buffer is the /// responsibility of the caller. /// \returns shared_ptr to a new backend-specific tensor - virtual std::shared_ptr create_tensor(const ngraph::element::Type& element_type, - const Shape& shape, - void* memory_pointer) = 0; + virtual ov::Tensor create_tensor(const ov::element::Type& element_type, + const Shape& shape, + void* memory_pointer) = 0; /// \brief Create a tensor of C type T specific to this backend /// \param shape The shape of the tensor /// \returns shared_ptr to a new backend specific tensor template - std::shared_ptr create_tensor(const Shape& shape) { + ov::Tensor create_tensor(const Shape& shape) { return create_tensor(element::from(), shape); } - /// \brief Create a dynamic tensor specific to this backend, if the backend supports dynamic - /// tensors. - /// \param element_type The type of the tensor element - /// \param shape The shape of the tensor - /// \returns shared_ptr to a new backend-specific tensor - /// \throws std::invalid_argument if the backend does not support dynamic tensors - virtual std::shared_ptr create_dynamic_tensor(const ngraph::element::Type& element_type, - const PartialShape& shape); - - /// \returns `true` if this backend supports dynamic tensors, else `false`. - virtual bool supports_dynamic_tensors() { - return false; - } /// \brief Compiles a Function. /// \param func The function to compile /// \returns compiled function or nullptr on failure - virtual std::shared_ptr compile(std::shared_ptr func, - bool enable_performance_data = false) = 0; - - /// \brief Loads a previously saved Executable object from a stream. - /// \param input_stream the opened input stream containing the saved Executable - /// \returns A compiled function or throws an exception on error - virtual std::shared_ptr load(std::istream& input_stream); - - /// \brief Test if a backend is capable of supporting an op - /// \param node is the op to test. - /// \returns true if the op is supported, false otherwise. - virtual bool is_supported(const Node& node) const; - - /// \brief Allows sending backend specific configuration. The map contains key, value pairs - /// specific to a particluar backend. The definition of these key, value pairs is - /// defined by each backend. - /// \param config The configuration map sent to the backend - /// \param error An error string describing any error encountered - /// \returns true if the configuration is supported, false otherwise. On false the error - /// parameter value is valid. - virtual bool set_config(const std::map& config, std::string& error); - - /// \brief Get the version of the backend - /// The default value of 0.0.0 is chosen to be a parsable version number - virtual std::string get_version() const { - return "0.0.0"; - } + virtual std::shared_ptr compile(std::shared_ptr model) = 0; }; + +} // namespace runtime +} // namespace ov diff --git a/src/plugins/template/backend/cache.cpp b/src/plugins/template/backend/cache.cpp deleted file mode 100644 index 0343ec57ff2..00000000000 --- a/src/plugins/template/backend/cache.cpp +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "cache.hpp" - -#include "ngraph/env_util.hpp" - -using namespace ngraph; -using namespace std; - -// Constructor -runtime::LRUCache::LRUCache() { - m_cache_size = 1024; - - m_map = {}; - m_list = {}; -} - -// Destructor -runtime::LRUCache::~LRUCache() { - m_list.clear(); - m_map.clear(); - m_clone_function_map.clear(); -} - -void runtime::LRUCache::convert_shape_to_string(const vector& shape, ostringstream& key) { - if (!shape.empty()) { - std::copy(shape.begin(), shape.end(), std::ostream_iterator(key, ", ")); - } -} - -void runtime::LRUCache::add_entry(const vector& shape, - shared_ptr exec, - shared_ptr func) { - std::lock_guard guard(m_mutex); - ostringstream key; - // check if the list is empty - if (m_list.size() == static_cast(m_cache_size)) { - ostringstream key; - convert_shape_to_string(m_list.back(), key); - m_list.pop_back(); - m_map.erase(key.str()); - } - - convert_shape_to_string(shape, key); - m_map.insert({key.str(), exec}); - m_list.push_front(shape); - m_clone_function_map.insert({key.str(), func}); -} - -bool runtime::LRUCache::is_cached(const vector& shape) { - for (auto itr = m_list.begin(); itr != m_list.end(); itr++) { - if (*itr == shape) { - return true; - } - } - return false; -} - -shared_ptr runtime::LRUCache::get_cached_entry(const vector& shape) { - std::lock_guard guard(m_mutex); - ostringstream key; - convert_shape_to_string(shape, key); - - // find the entry and return the function - auto it = m_map.find(key.str()); - if (it == m_map.end()) { - throw ngraph_error("Entry not found in cache"); - } else { - // update list to push this reference to the front - for (auto itr = m_list.begin(); itr != m_list.end(); itr++) { - if (*itr == shape) { - m_list.remove(shape); - m_list.push_front(shape); - break; - } - } - return it->second; - } -} - -// Need the clone function to get the output shape so that -// storage can be allocated for output -shared_ptr runtime::LRUCache::get_cloned_function(const vector& shape) { - std::lock_guard guard(m_mutex); - ostringstream key; - convert_shape_to_string(shape, key); - // find the entry and return the function - auto it = m_clone_function_map.find(key.str()); - if (it == m_clone_function_map.end()) { - throw ngraph_error("Cloned function not found"); - } - return it->second; -} diff --git a/src/plugins/template/backend/cache.hpp b/src/plugins/template/backend/cache.hpp deleted file mode 100644 index aef64cccec8..00000000000 --- a/src/plugins/template/backend/cache.hpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "executable.hpp" -#include "ngraph/function.hpp" -#include "ngraph/shape.hpp" - -namespace ngraph { -namespace runtime { -class LRUCache : public std::enable_shared_from_this { -public: - using GraphCache = std::unordered_map>; - using ClonedFunctionMap = std::unordered_map>; - - LRUCache(); - - virtual ~LRUCache(); - - void add_entry(const std::vector& shape, std::shared_ptr exec, std::shared_ptr func); - bool is_cached(const std::vector& shape); - std::shared_ptr get_cached_entry(const std::vector& shape); - void convert_shape_to_string(const std::vector& shape, std::ostringstream& key); - std::shared_ptr get_cloned_function(const std::vector& shape); - -private: - int m_cache_size; - GraphCache m_map; - ClonedFunctionMap m_clone_function_map; - std::list> m_list; - std::mutex m_mutex; -}; -} // namespace runtime -} // namespace ngraph diff --git a/src/plugins/template/backend/evaluates_map.cpp b/src/plugins/template/backend/evaluates_map.cpp index 2d8c4b45a1b..7b4f95af625 100644 --- a/src/plugins/template/backend/evaluates_map.cpp +++ b/src/plugins/template/backend/evaluates_map.cpp @@ -95,6 +95,7 @@ #include "ngraph/runtime/reference/convert_color_nv12.hpp" #include "ov_ops/augru_cell.hpp" #include "ov_ops/augru_sequence.hpp" +#include "tensor_conversion_util.hpp" using namespace ngraph; using namespace std; @@ -3302,46 +3303,21 @@ runtime::reference::custom_evaluate_function evaluate = [](const std::shared_ptr inputsNumber, " input blobs"); - auto inputTensors = std::vector>{}; - for (const auto& parameter : parameters) { - const auto& parameterIndex = function->get_parameter_index(parameter); - const auto& parameterShape = parameter->get_shape(); - const auto& parameterType = parameter->get_element_type(); - const auto& parameterSize = shape_size(parameterShape) * parameterType.size(); - - const auto& input = inputs[parameterIndex]; - const auto& inputSize = input->get_size_in_bytes(); - NGRAPH_CHECK(parameterSize == inputSize, - "Got parameter (", - parameter->get_friendly_name(), - ") of size ", - parameterSize, - " bytes, but corresponding input with index ", - parameterIndex, - " has ", - inputSize, - " bytes"); - - auto tensor = std::make_shared(parameterType, parameterShape); - tensor->write(input->get_data_ptr(), parameterSize); - inputTensors.push_back(tensor); - } - const auto& results = function->get_results(); - std::vector> outputTensors; - outputTensors.reserve(results.size()); + outputs.reserve(results.size()); for (size_t i = 0; i < results.size(); ++i) { - outputTensors.push_back(std::make_shared()); + outputs.push_back(std::make_shared(results[i]->output(0))); } - auto backend = runtime::Backend::create(); + + auto backend = ov::runtime::Backend::create(); auto handle = backend->compile(function); + OPENVINO_SUPPRESS_DEPRECATED_START + auto outputTensors = ov::util::wrap_tensors(outputs); + auto inputTensors = ov::util::wrap_tensors(inputs); handle->call_with_validate(outputTensors, inputTensors); - outputs.reserve(outputTensors.size()); - for (const auto& tensor : outputTensors) { - auto host_tensor = static_pointer_cast(tensor); - outputs.push_back(host_tensor); - } + ov::util::update_output_host_tensors(outputs, outputTensors); + OPENVINO_SUPPRESS_DEPRECATED_END }; } // namespace ti_v0 diff --git a/src/plugins/template/backend/executable.cpp b/src/plugins/template/backend/executable.cpp index 883c3cbdc03..61e39a76881 100644 --- a/src/plugins/template/backend/executable.cpp +++ b/src/plugins/template/backend/executable.cpp @@ -6,133 +6,81 @@ #include -#include "ngraph/file_util.hpp" -#include "ngraph/runtime/tensor.hpp" -#include "ngraph/util.hpp" +#include "openvino/core/except.hpp" -using namespace std; -using namespace ngraph; +ov::runtime::Executable::Executable() {} -runtime::Executable::Executable() {} +ov::runtime::Executable::~Executable() {} -runtime::Executable::~Executable() {} - -bool runtime::Executable::call_with_validate(const vector>& outputs, - const vector>& inputs) { +bool ov::runtime::Executable::call_with_validate(std::vector& outputs, + const std::vector& inputs) { validate(outputs, inputs); return call(outputs, inputs); } -void runtime::Executable::validate(const vector>& outputs, - const vector>& inputs) { +void ov::runtime::Executable::validate(const std::vector& outputs, const std::vector& inputs) { const ParameterVector& parameters = get_parameters(); const ResultVector& results = get_results(); if (parameters.size() != inputs.size()) { - stringstream ss; + std::stringstream ss; ss << "Call input count " << inputs.size() << " does not match Function's Parameter count " << parameters.size(); - throw runtime_error(ss.str()); + throw std::runtime_error(ss.str()); } if (results.size() != outputs.size()) { - stringstream ss; + std::stringstream ss; ss << "Call output count " << outputs.size() << " does not match Function's Result count " << results.size(); - throw runtime_error(ss.str()); + throw std::runtime_error(ss.str()); } for (size_t i = 0; i < parameters.size(); i++) { if (parameters[i]->get_element_type().is_static() && - parameters[i]->get_element_type() != inputs[i]->get_element_type()) { - stringstream ss; - ss << "Input " << i << " type '" << inputs[i]->get_element_type() << "' does not match Parameter type '" + parameters[i]->get_element_type() != inputs[i].get_element_type()) { + std::stringstream ss; + ss << "Input " << i << " type '" << inputs[i].get_element_type() << "' does not match Parameter type '" << parameters[i]->get_element_type() << "'"; - throw runtime_error(ss.str()); - } - if (!(parameters[i]->get_output_partial_shape(0).relaxes(inputs[i]->get_partial_shape()))) { - stringstream ss; - ss << "Input " << i << " shape " << inputs[i]->get_partial_shape() << " does not match Parameter shape " - << parameters[i]->get_output_partial_shape(0); - throw runtime_error(ss.str()); + throw std::runtime_error(ss.str()); } } for (size_t i = 0; i < results.size(); i++) { - if (outputs[i]->get_element_type().is_static() && results[i]->get_element_type().is_static() && - results[i]->get_element_type() != outputs[i]->get_element_type()) { - stringstream ss; - ss << "Output " << i << " type '" << outputs[i]->get_element_type() << "' does not match Result type '" + if (outputs[i].get_element_type().is_static() && results[i]->get_element_type().is_static() && + results[i]->get_element_type() != outputs[i].get_element_type()) { + std::stringstream ss; + ss << "Output " << i << " type '" << outputs[i].get_element_type() << "' does not match Result type '" << results[i]->get_element_type() << "'"; - throw runtime_error(ss.str()); - } - if (!outputs[i]->get_partial_shape().relaxes(results[i]->get_output_partial_shape(0))) { - stringstream ss; - ss << "Output " << i << " shape " << outputs[i]->get_partial_shape() << " does not match max Result shape " - << results[i]->get_output_partial_shape(0).get_max_shape(); - throw runtime_error(ss.str()); + throw std::runtime_error(ss.str()); } } } -const ngraph::ParameterVector& runtime::Executable::get_parameters() const { +const ov::ParameterVector& ov::runtime::Executable::get_parameters() const { return m_parameters; } -const ngraph::ResultVector& runtime::Executable::get_results() const { +const ov::ResultVector& ov::runtime::Executable::get_results() const { return m_results; } -size_t runtime::Executable::get_preferred_pipeline_depth() const { - return 2; +void ov::runtime::Executable::set_parameters_and_results(const ov::Model& model) { + m_parameters = model.get_parameters(); + m_results = model.get_results(); } -void runtime::Executable::set_parameters_and_results(const Function& func) { - m_parameters = func.get_parameters(); - m_results = func.get_results(); +ov::Tensor ov::runtime::Executable::create_input_tensor(size_t /* input_index */) { + OPENVINO_NOT_IMPLEMENTED; } -vector runtime::Executable::get_performance_data() const { - return vector(); +ov::Tensor ov::runtime::Executable::create_output_tensor(size_t /* output_index */) { + OPENVINO_NOT_IMPLEMENTED; } -void runtime::Executable::save(std::ostream& /* output_stream */) { - throw runtime_error("save operation unimplemented."); +std::vector ov::runtime::Executable::create_input_tensor(size_t /* input_index */, + size_t /* pipeline_depth */) { + OPENVINO_NOT_IMPLEMENTED; } -shared_ptr runtime::Executable::create_input_tensor(size_t /* input_index */) { - throw runtime_error("create_input_tensor unimplemented"); -} - -shared_ptr runtime::Executable::create_input_tensor(size_t /* input_index */, - void* /* memory_pointer */) { - throw runtime_error("create_input_tensor unimplemented"); -} - -shared_ptr runtime::Executable::create_output_tensor(size_t /* output_index */) { - throw runtime_error("create_output_tensor unimplemented"); -} - -shared_ptr runtime::Executable::create_output_tensor(size_t /* output_index */, - void* /* memory_pointer */) { - throw runtime_error("create_output_tensor unimplemented"); -} - -vector> runtime::Executable::create_input_tensor(size_t /* input_index */, - size_t /* pipeline_depth */) { - throw runtime_error("create_input_tensor unimplemented"); -} - -vector> runtime::Executable::create_input_tensor(size_t /* input_index */, - size_t /* pipeline_depth */, - std::vector /* memory_pointer */) { - throw runtime_error("create_input_tensor unimplemented"); -} - -vector> runtime::Executable::create_output_tensor(size_t /* output_index */, - size_t /* pipeline_depth */) { - throw runtime_error("create_output_tensor unimplemented"); -} - -vector> runtime::Executable::create_output_tensor(size_t /* output_index */, - size_t /* pipeline_depth */, - std::vector /* memory_pointer */) { - throw runtime_error("create_output_tensor unimplemented"); +std::vector ov::runtime::Executable::create_output_tensor(size_t /* output_index */, + size_t /* pipeline_depth */) { + OPENVINO_NOT_IMPLEMENTED; } diff --git a/src/plugins/template/backend/executable.hpp b/src/plugins/template/backend/executable.hpp index 036e4f2de02..2375d6e0a09 100644 --- a/src/plugins/template/backend/executable.hpp +++ b/src/plugins/template/backend/executable.hpp @@ -6,19 +6,15 @@ #include -#include "ngraph/function.hpp" -#include "ngraph/runtime/tensor.hpp" -#include "ngraph/shape.hpp" -#include "ngraph/type/element_type.hpp" -#include "performance_counter.hpp" +#include "openvino/core/model.hpp" +#include "openvino/core/node_vector.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/runtime/tensor.hpp" -namespace ngraph { +namespace ov { namespace runtime { -class Executable; -} -} // namespace ngraph -class ngraph::runtime::Executable { +class Executable { public: Executable(); virtual ~Executable(); @@ -26,71 +22,38 @@ public: /// \param outputs vector of runtime::Tensor used as outputs /// \param inputs vector of runtime::Tensor used as inputs /// \returns true if iteration is successful, false otherwise - virtual bool call(const std::vector>& outputs, - const std::vector>& inputs) = 0; + virtual bool call(std::vector& outputs, const std::vector& inputs) = 0; /// \brief Executes a single iteration of a Function. /// \param outputs vector of runtime::Tensor used as outputs /// \param inputs vector of runtime::Tensor used as inputs /// \returns true if iteration is successful, false otherwise - bool call_with_validate(const std::vector>& outputs, - const std::vector>& inputs); - - /// \brief Collect performance information gathered on a Function. - /// \returns Vector of PerformanceCounter information. - virtual std::vector get_performance_data() const; + bool call_with_validate(std::vector& outputs, const std::vector& inputs); /// \brief Validates a Function. /// \param outputs vector of runtime::Tensor used as outputs /// \param inputs vector of runtime::Tensor used as inputs - void validate(const std::vector>& outputs, - const std::vector>& inputs); + void validate(const std::vector& outputs, const std::vector& inputs); /// \brief Query the input Parameters /// \returns an ngraph::op::ParameterVector of all input parameters - const ngraph::ParameterVector& get_parameters() const; + const ov::ParameterVector& get_parameters() const; /// \brief Query the output Results /// \returns an ngraph::ResultVector of all input parameters - const ngraph::ResultVector& get_results() const; - - /// \brief Get the preferred pipeline_depth for this executable - /// \returns preferred pipeline_depth - virtual size_t get_preferred_pipeline_depth() const; - - /// \brief Save this compiled Executable to an output stream. - /// Saved stream may be read with Backend::load - virtual void save(std::ostream& output_stream); + const ov::ResultVector& get_results() const; /// \brief Create an input Tensor /// \param input_index The index position in the input Parameter vector. This would be the same /// order of Parameters passed into the inputs in the call() method. /// \returns A Tensor - virtual std::shared_ptr create_input_tensor(size_t input_index); - - /// \brief Create an input Tensor - /// \param input_index The index position in the input Parameter vector. This would be the same - /// order of Parameters passed into the inputs in the call() method. - /// \param memory_pointer A pointer to a buffer used for this tensor. The size of the buffer - /// must be sufficient to contain the tensor. The lifetime of the buffer is the - /// responsibility of the caller and must outlive the created Tensor. - /// \returns A Tensor - virtual std::shared_ptr create_input_tensor(size_t input_index, void* memory_pointer); + virtual ov::Tensor create_input_tensor(size_t input_index); /// \brief Create an output Tensor /// \param output_index The index position in the output Result vector. This would be the same /// order of Results passed into the outputs in the call() method. /// \returns A Tensor - virtual std::shared_ptr create_output_tensor(size_t output_index); - - /// \brief Create an output Tensor - /// \param output_index The index position in the output Result vector. This would be the same - /// order of Results passed into the outputs in the call() method. - /// \param memory_pointer A pointer to a buffer used for this tensor. The size of the buffer - /// must be sufficient to contain the tensor. The lifetime of the buffer is the - /// responsibility of the caller and must outlive the created Tensor. - /// \returns A Tensor - virtual std::shared_ptr create_output_tensor(size_t output_index, void* memory_pointer); + virtual ov::Tensor create_output_tensor(size_t output_index); /// \brief Create a vector of input Tensors /// \param input_index The index position in the input Parameter vector. This would be the same @@ -98,21 +61,7 @@ public: /// \param pipeline_depth The number of stages in the input pipeline. For double-buffered input /// you would specify pipeline_depth=2 /// \returns A vector of Tensors, one for each stage of the pipeline - virtual std::vector> create_input_tensor(size_t input_index, - size_t pipeline_depth); - - /// \brief Create a vector of input Tensors - /// \param input_index The index position in the input Parameter vector. This would be the same - /// order of Parameters passed into the inputs in the call() method. - /// \param pipeline_depth The number of stages in the input pipeline. For double-buffered input - /// you would specify pipeline_depth=2 - /// \param memory_pointers A vector of pointers to buffers used for this tensors. The size of - /// the buffer must be sufficient to contain the tensor. The lifetime of the buffers is the - /// responsibility of the caller and must outlive the created Tensor. - /// \returns A vector of Tensors, one for each stage of the pipeline - virtual std::vector> create_input_tensor(size_t input_index, - size_t pipeline_depth, - std::vector memory_pointers); + virtual std::vector create_input_tensor(size_t input_index, size_t pipeline_depth); /// \brief Create a vector of output Tensors /// \param output_index The index position in the output Result vector. This would be the same @@ -120,28 +69,17 @@ public: /// \param pipeline_depth The number of stages in the output pipeline. For double-buffered /// output you would specify pipeline_depth=2 /// \returns A vector of Tensors, one for each stage of the pipeline - virtual std::vector> create_output_tensor(size_t output_index, - size_t pipeline_depth); - - /// \brief Create a vector of output Tensors - /// \param output_index The index position in the output Result vector. This would be the same - /// order of Results passed into the outputs in the call() method. - /// \param pipeline_depth The number of stages in the output pipeline. For double-buffered - /// output you would specify pipeline_depth=2 - /// \param memory_pointers A vector of pointers to buffers used for this tensors. The size of - /// the buffer must be sufficient to contain the tensor. The lifetime of the buffers is the - /// responsibility of the caller and must outlive the created Tensor. - /// \returns A vector of Tensors, one for each stage of the pipeline - virtual std::vector> create_output_tensor(size_t output_index, - size_t pipeline_depth, - std::vector memory_pointers); + virtual std::vector create_output_tensor(size_t output_index, size_t pipeline_depth); protected: /// \brief Called at the end of compile to the values to be returned by get_parameters /// and get_results /// \param func The function with Results fully resolved. - void set_parameters_and_results(const Function& func); + void set_parameters_and_results(const ov::Model& model); - ngraph::ParameterVector m_parameters; - ngraph::ResultVector m_results; + ov::ParameterVector m_parameters; + ov::ResultVector m_results; }; + +} // namespace runtime +} // namespace ov diff --git a/src/plugins/template/backend/int_backend.cpp b/src/plugins/template/backend/int_backend.cpp index cf1b270a2cc..0d75baea7d7 100644 --- a/src/plugins/template/backend/int_backend.cpp +++ b/src/plugins/template/backend/int_backend.cpp @@ -5,54 +5,27 @@ #include "int_backend.hpp" #include "int_executable.hpp" -#include "ngraph/except.hpp" -#include "ngraph/runtime/host_tensor.hpp" -#include "ngraph/util.hpp" -using namespace std; -using namespace ngraph; +ov::runtime::interpreter::INTBackend::INTBackend() {} -runtime::interpreter::INTBackend::INTBackend() {} - -runtime::interpreter::INTBackend::INTBackend(const vector& unsupported_op_name_list) +ov::runtime::interpreter::INTBackend::INTBackend(const std::vector& unsupported_op_name_list) : m_unsupported_op_name_list{unsupported_op_name_list.begin(), unsupported_op_name_list.end()} {} -shared_ptr runtime::interpreter::INTBackend::create_tensor() { - return make_shared(); +ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor() { + return ov::Tensor(); } -shared_ptr runtime::interpreter::INTBackend::create_tensor(const element::Type& type, - const Shape& shape) { - return make_shared(type, shape); +ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor(const element::Type& type, const Shape& shape) { + return ov::Tensor(type, shape); } -shared_ptr runtime::interpreter::INTBackend::create_dynamic_tensor(const element::Type& type, - const PartialShape& pshape) { - return make_shared(type, pshape); +ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor(const element::Type& type, + const Shape& shape, + void* memory_pointer) { + return ov::Tensor(type, shape, memory_pointer); } -shared_ptr runtime::interpreter::INTBackend::create_tensor(const element::Type& type, - const Shape& shape, - void* memory_pointer) { - return make_shared(type, shape, memory_pointer); -} - -shared_ptr runtime::interpreter::INTBackend::compile(shared_ptr function, - bool enable_performance_collection) { - return make_shared(function, enable_performance_collection); -} - -bool runtime::interpreter::INTBackend::is_supported(const Node& node) const { - return m_unsupported_op_name_list.find(node.description()) == m_unsupported_op_name_list.end(); -} - -bool runtime::interpreter::INTBackend::set_config(const map& config, string& error) { - bool rc = false; - auto it = config.find("test_echo"); - error = ""; - if (it != config.end()) { - error = it->second; - rc = true; - } - return rc; +std::shared_ptr ov::runtime::interpreter::INTBackend::compile( + std::shared_ptr model) { + return std::make_shared(model); } diff --git a/src/plugins/template/backend/int_backend.hpp b/src/plugins/template/backend/int_backend.hpp index b688fa151b0..ac45776d2e2 100644 --- a/src/plugins/template/backend/int_backend.hpp +++ b/src/plugins/template/backend/int_backend.hpp @@ -11,18 +11,13 @@ #include #include "backend.hpp" -#include "ngraph/runtime/tensor.hpp" +#include "openvino/core/model.hpp" -namespace ngraph { +namespace ov { namespace runtime { namespace interpreter { -class INTBackend; -class INTExecutable; -} // namespace interpreter -} // namespace runtime -} // namespace ngraph -class ngraph::runtime::interpreter::INTBackend : public Backend { +class INTBackend : public Backend { public: INTBackend(); INTBackend(const std::vector& unsupported_op_name_list); @@ -30,20 +25,18 @@ public: INTBackend(INTBackend&&) = delete; INTBackend& operator=(const INTBackend&) = delete; - std::shared_ptr create_tensor() override; + ov::Tensor create_tensor() override; - std::shared_ptr create_tensor(const element::Type& type, const Shape& shape, void* memory_pointer) override; + ov::Tensor create_tensor(const element::Type& type, const Shape& shape, void* memory_pointer) override; - std::shared_ptr create_tensor(const element::Type& type, const Shape& shape) override; - std::shared_ptr create_dynamic_tensor(const element::Type& type, const PartialShape& shape) override; + ov::Tensor create_tensor(const element::Type& type, const Shape& shape) override; - std::shared_ptr compile(std::shared_ptr function, - bool enable_performance_data = false) override; - - bool is_supported(const Node& node) const override; - - bool set_config(const std::map& config, std::string& error) override; + std::shared_ptr compile(std::shared_ptr model) override; private: std::set m_unsupported_op_name_list; }; + +} // namespace interpreter +} // namespace runtime +} // namespace ov diff --git a/src/plugins/template/backend/int_executable.cpp b/src/plugins/template/backend/int_executable.cpp index b51ad5b0023..fc4057955fe 100644 --- a/src/plugins/template/backend/int_executable.cpp +++ b/src/plugins/template/backend/int_executable.cpp @@ -5,32 +5,85 @@ #include "int_executable.hpp" #include +#include #include #include "evaluates_map.hpp" -#include "ngraph/except.hpp" -#include "ngraph/ops.hpp" -#include "ngraph/type/bfloat16.hpp" -#include "ngraph/type/float16.hpp" -#include "ngraph/util.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/result.hpp" +#include "openvino/op/util/op_types.hpp" #include "tensor_conversion_util.hpp" -using namespace std; -using namespace ngraph; - NGRAPH_SUPPRESS_DEPRECATED_START -class TemporaryOverrideOutputs { - std::shared_ptr node; - std::vector orig_shapes; +namespace { + +class DynamicTensor : public ngraph::runtime::HostTensor { +private: + ov::Tensor tensor; public: - TemporaryOverrideOutputs(std::shared_ptr node, const std::vector>& args) - : node(node) { + DynamicTensor(const ov::element::Type& type) : ngraph::runtime::HostTensor(type, ov::PartialShape::dynamic()) {} + + ov::Tensor get_tensor() { + return tensor; + } + +protected: + void allocate_buffer() override { + OPENVINO_ASSERT(get_partial_shape().is_static(), + "Attempt to allocate buffer for tensor with partial shape: ", + get_partial_shape()); + OPENVINO_ASSERT(get_element_type().is_static(), + "Attempt to allocate buffer for tensor with dynamic type: ", + get_element_type()); + m_buffer_size = m_descriptor->size(); + tensor = ov::Tensor(get_element_type(), get_partial_shape().get_shape()); + m_memory_pointer = tensor.data(); + m_aligned_buffer_pool = m_memory_pointer; + } +}; + +inline ngraph::HostTensorPtr make_tmp_host_tensor(const ov::Tensor& t) { + OPENVINO_SUPPRESS_DEPRECATED_START + if (!t) { + return std::make_shared(ov::element::dynamic); + } else if (t.get_shape() == ov::Shape{0, std::numeric_limits::max()}) { + return std::make_shared(t.get_element_type()); + } else { + return std::make_shared(t.get_element_type(), t.get_shape(), t.data()); + } + OPENVINO_SUPPRESS_DEPRECATED_END +} +inline ngraph::HostTensorVector create_tmp_tensors(const ov::TensorVector& tensors) { + ngraph::HostTensorVector result; + result.reserve(tensors.size()); + for (const auto& tensor : tensors) { + result.push_back(make_tmp_host_tensor(tensor)); + } + return result; +} + +inline void update_output_tensors(ov::TensorVector& output_values, const ngraph::HostTensorVector& outputs) { + OPENVINO_ASSERT(output_values.size() == outputs.size()); + for (size_t i = 0; i < outputs.size(); i++) { + if (auto dyn_output = std::dynamic_pointer_cast(outputs[i])) { + output_values[i] = dyn_output->get_tensor(); + } + } +} +} // namespace + +class TemporaryOverrideOutputs { + std::shared_ptr node; + std::vector orig_shapes; + +public: + TemporaryOverrideOutputs(std::shared_ptr node, const std::vector& args) : node(node) { for (size_t i = 0; i < args.size(); ++i) { auto output = node->get_input_source_output(i); orig_shapes.push_back(output.get_partial_shape()); - output.get_tensor().set_partial_shape(args[i]->get_shape()); + output.get_tensor().set_partial_shape(args[i].get_shape()); } } @@ -42,43 +95,23 @@ public: } }; -runtime::interpreter::INTExecutable::INTExecutable(const shared_ptr& function, - bool enable_performance_collection) - : m_is_compiled{true}, - m_performance_counters_enabled{enable_performance_collection} { - m_function = clone_function(*function); - for (auto node : m_function->get_ordered_ops()) { +ov::runtime::interpreter::INTExecutable::INTExecutable(const std::shared_ptr& model) : m_is_compiled{true} { + m_model = model->clone(); + for (auto node : m_model->get_ordered_ops()) { m_nodes.push_back(node); } - set_parameters_and_results(*m_function); + set_parameters_and_results(*m_model); } -bool runtime::interpreter::INTExecutable::call(const vector>& outputs, - const vector>& inputs) { - // convert inputs to HostTensor - vector> func_inputs; - for (const auto& tensor : inputs) { - auto host_tensor = static_pointer_cast(tensor); - func_inputs.push_back(host_tensor); - } - if (m_nan_check_enabled) { - perform_nan_check(func_inputs); - } - - // convert outputs to HostTensor - vector> func_outputs; - for (const auto& tensor : outputs) { - auto host_tensor = static_pointer_cast(tensor); - func_outputs.push_back(host_tensor); - } - +bool ov::runtime::interpreter::INTExecutable::call(std::vector& outputs, + const std::vector& inputs) { // map function params -> HostTensor - std::unordered_map, shared_ptr> tensor_map; + std::unordered_map, ov::Tensor> tensor_map; size_t input_count = 0; for (const auto& param : get_parameters()) { for (size_t i = 0; i < param->get_output_size(); ++i) { auto tensor = param->output(i).get_tensor_ptr(); - tensor_map.insert({tensor, func_inputs[input_count++]}); + tensor_map.insert({tensor, inputs[input_count++]}); } } @@ -96,186 +129,138 @@ bool runtime::interpreter::INTExecutable::call(const vector(op) != nullptr) { + if (std::dynamic_pointer_cast(op)) { continue; } // get op inputs from map - vector> op_inputs; + std::vector op_inputs; for (auto input : op->inputs()) { auto tensor = input.get_tensor_ptr(); op_inputs.push_back(tensor_map.at(tensor)); } TemporaryOverrideOutputs overrider(op, op_inputs); - OutputVector outputs; + OutputVector output_ports; for (size_t i = 0; i < op->inputs().size(); ++i) { - outputs.push_back(op->get_input_source_output(i)); + output_ports.push_back(op->get_input_source_output(i)); } - auto cloned_node = op->clone_with_new_inputs(outputs); + auto cloned_node = op->clone_with_new_inputs(output_ports); // get op outputs from map or create - vector> op_outputs; + std::vector op_outputs; for (size_t i = 0; i < op->get_output_size(); ++i) { auto tensor = op->output(i).get_tensor_ptr(); - shared_ptr host_tensor; + ov::Tensor host_tensor; auto it = tensor_map.find(tensor); - if (op::is_output(op)) { - host_tensor = func_outputs[results_map[tensor]]; - } else if (it == tensor_map.end()) { - // Use cloned_node to create HostTensor with static dimensions - host_tensor = make_shared(cloned_node->output(i)); - tensor_map.insert({tensor, host_tensor}); + auto output = cloned_node->output(i); + if (op::util::is_output(op) || it == tensor_map.end() || !it->second) { + host_tensor = ov::Tensor(output.get_element_type(), + output.get_partial_shape().is_dynamic() + ? ov::Shape{0, std::numeric_limits::max()} + : output.get_shape()); } else { host_tensor = it->second; } op_outputs.push_back(host_tensor); } - if (m_performance_counters_enabled) { - m_timer_map[op].start(); - } - if (auto var_extension = std::dynamic_pointer_cast(cloned_node)) { auto variable = var_extension->get_variable(); if (!variable_context.get_variable_value(variable)) { - auto h_tensor = std::make_shared(cloned_node->get_input_element_type(0), - cloned_node->get_input_shape(0)); - h_tensor->write(h_tensor->get_data_ptr(), h_tensor->get_size_in_bytes()); - variable_context.set_variable_value(variable, std::make_shared(h_tensor)); + auto h_tensor = ov::Tensor(cloned_node->get_input_element_type(0), cloned_node->get_input_shape(0)); + // h_tensor->write(h_tensor->get_data_ptr(), h_tensor->get_size_in_bytes()); + const auto tensor_input = make_tmp_host_tensor(h_tensor); + variable_context.set_variable_value(variable, + std::make_shared(tensor_input)); } } - const auto tensor_inputs = ov::util::wrap_tensors(op_inputs); - auto tensor_outputs = ov::util::wrap_tensors(op_outputs); - // Call evaluate for cloned_node with static shapes - if (cloned_node->evaluate(tensor_outputs, tensor_inputs, eval_context)) { - ov::util::update_output_host_tensors(op_outputs, tensor_outputs); - } else { + if (!cloned_node->evaluate(op_outputs, op_inputs, eval_context)) { evaluate_node(cloned_node, op_outputs, op_inputs); } - if (m_performance_counters_enabled) { - m_timer_map[op].stop(); - } - if (m_nan_check_enabled) { - perform_nan_check(op_outputs, op.get()); + // Update tensors in tensor map + for (size_t i = 0; i < op->get_output_size(); ++i) { + auto tensor = op->output(i).get_tensor_ptr(); + tensor_map.insert({tensor, op_outputs[i]}); + if (op::util::is_output(op)) { + auto& output = outputs[results_map[tensor]]; + if (!output || output.get_shape() != op_outputs[i].get_shape()) { + outputs[results_map[tensor]] = op_outputs[i]; + } else { + op_outputs[i].copy_to(output); + } + } } } return true; } -vector runtime::interpreter::INTExecutable::get_performance_data() const { - vector rc; - for (const pair, stopwatch> p : m_timer_map) { - rc.emplace_back(p.first, p.second.get_total_microseconds(), p.second.get_call_count()); - } - return rc; -} - -void runtime::interpreter::INTExecutable::perform_nan_check(const vector>& tensors, - const Node* op) { - size_t arg_number = 1; - for (const shared_ptr& tensor : tensors) { - const element::Type& type = tensor->get_element_type(); - if (type == element::f32) { - const float* data = tensor->get_data_ptr(); - for (size_t i = 0; i < tensor->get_element_count(); i++) { - if (std::isnan(data[i])) { - if (op) { - throw runtime_error("nan found in op '" + op->get_name() + "' output"); - } else { - throw runtime_error("nan found in function's input tensor number " + to_string(arg_number)); - } - } - } - } else if (type == element::f64) { - const double* data = tensor->get_data_ptr(); - for (size_t i = 0; i < tensor->get_element_count(); i++) { - if (std::isnan(data[i])) { - if (op) { - throw runtime_error("nan found in op '" + op->get_name() + "' output"); - } else { - throw runtime_error("nan found in function's input tensor number " + to_string(arg_number)); - } - } - } - } - arg_number++; - } -} - -shared_ptr runtime::interpreter::INTExecutable::get_parameter(size_t index) const { +std::shared_ptr ov::runtime::interpreter::INTExecutable::get_parameter(size_t index) const { const ParameterVector& parameters = get_parameters(); NGRAPH_CHECK(index < parameters.size(), "create_tensor for input out of bounds"); return parameters[index]; } -shared_ptr runtime::interpreter::INTExecutable::get_result(size_t index) const { +std::shared_ptr ov::runtime::interpreter::INTExecutable::get_result(size_t index) const { const ResultVector& results = get_results(); NGRAPH_CHECK(index < results.size(), "create_tensor for input out of bounds"); return results[index]; } -shared_ptr runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index) { - shared_ptr parameter = get_parameter(input_index); - return make_shared(parameter->get_element_type(), parameter->get_shape()); +ov::Tensor ov::runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index) { + std::shared_ptr parameter = get_parameter(input_index); + return ov::Tensor(parameter->get_element_type(), parameter->get_shape()); } -shared_ptr runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index) { - shared_ptr result = get_result(output_index); - return make_shared(result->get_element_type(), result->get_shape()); +ov::Tensor ov::runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index) { + std::shared_ptr result = get_result(output_index); + return ov::Tensor(result->get_element_type(), result->get_shape()); } -vector> runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index, - size_t pipeline_depth) { - vector> tensors; - shared_ptr parameter = get_parameter(input_index); +std::vector ov::runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index, + size_t pipeline_depth) { + std::vector tensors; + std::shared_ptr parameter = get_parameter(input_index); for (size_t i = 0; i < pipeline_depth; i++) { - shared_ptr tensor; - auto t = make_shared(parameter->get_element_type(), parameter->get_shape()); - tensor = static_pointer_cast(t); - tensors.push_back(tensor); + ov::Tensor tensor; + auto t = ov::Tensor(parameter->get_element_type(), parameter->get_shape()); + tensors.push_back(t); } - vector> result_tensors; - for (const shared_ptr& tensor : tensors) { - result_tensors.push_back(tensor); - } - return result_tensors; + return tensors; } -vector> runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index, - size_t pipeline_depth) { - vector> tensors; - shared_ptr result = get_result(output_index); +std::vector ov::runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index, + size_t pipeline_depth) { + std::vector tensors; + std::shared_ptr result = get_result(output_index); for (size_t i = 0; i < pipeline_depth; i++) { - shared_ptr tensor; - auto t = make_shared(result->get_element_type(), result->get_shape()); - tensor = static_pointer_cast(t); - tensors.push_back(tensor); + ov::Tensor tensor; + auto t = ov::Tensor(result->get_element_type(), result->get_shape()); + tensors.push_back(t); } - vector> result_tensors; - for (const shared_ptr& tensor : tensors) { - result_tensors.push_back(tensor); - } - return result_tensors; + return tensors; } -bool runtime::interpreter::INTExecutable::evaluate_node(const std::shared_ptr& node, - const HostTensorVector& outputs, - const HostTensorVector& inputs) const { - auto& map = runtime::interpreter::get_evaluators_map(); +bool ov::runtime::interpreter::INTExecutable::evaluate_node(const std::shared_ptr& node, + ov::TensorVector& outputs, + const ov::TensorVector& inputs) const { + auto& map = ngraph::runtime::interpreter::get_evaluators_map(); auto it = map.find(node->get_type_info()); bool res = false; + const auto tensor_inputs = create_tmp_tensors(inputs); + auto tensor_outputs = create_tmp_tensors(outputs); if (it != map.end()) { - res = it->second(node, outputs, inputs); + res = it->second(node, tensor_outputs, tensor_inputs); if (!res) { - throw ngraph_error(std::string("Running evaluate method for OP ") + node->get_type_info().name + - std::string(" failed!")); + throw ngraph::ngraph_error(std::string("Running evaluate method for OP ") + node->get_type_info().name + + std::string(" failed!")); } + update_output_tensors(outputs, tensor_outputs); } else { - throw unsupported_op(std::string("Interpreter backend doesn't implement evaluate method for OP ") + - node->get_type_info().name); + throw ngraph::unsupported_op(std::string("Interpreter backend doesn't implement evaluate method for OP ") + + node->get_type_info().name); } return res; } diff --git a/src/plugins/template/backend/int_executable.hpp b/src/plugins/template/backend/int_executable.hpp index 2df580fa656..1ca49ff4253 100644 --- a/src/plugins/template/backend/int_executable.hpp +++ b/src/plugins/template/backend/int_executable.hpp @@ -7,68 +7,47 @@ #include #include #include -#include #include #include #include #include "backend.hpp" -#include "ngraph/ops.hpp" -#include "ngraph/runtime/aligned_buffer.hpp" -#include "ngraph/runtime/reference/hard_sigmoid.hpp" -#include "ngraph/runtime/reference/non_max_suppression.hpp" -#include "ngraph/runtime/reference/reorg_yolo.hpp" -#include "ngraph/runtime/reference/tensor_iterator.hpp" -#include "ngraph/runtime/tensor.hpp" +#include "openvino/core/model.hpp" +#include "openvino/op/non_max_suppression.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/result.hpp" -namespace ngraph { +namespace ov { namespace runtime { namespace interpreter { class INTBackend; -class INTExecutable; -} // namespace interpreter -} // namespace runtime -} // namespace ngraph -class ngraph::runtime::interpreter::INTExecutable : public Executable { +class INTExecutable : public Executable { friend class INTBackend; public: - INTExecutable(const std::shared_ptr& function, bool enable_performance_collection = false); + INTExecutable(const std::shared_ptr& model); - bool call(const std::vector>& outputs, - const std::vector>& inputs) override; + bool call(std::vector& outputs, const std::vector& inputs) override; - void set_nan_check(bool enable); + ov::Tensor create_input_tensor(size_t input_index) override; - std::vector get_performance_data() const override; + ov::Tensor create_output_tensor(size_t output_index) override; - std::shared_ptr create_input_tensor(size_t input_index) override; + std::vector create_input_tensor(size_t input_index, size_t pipeline_depth) override; - std::shared_ptr create_output_tensor(size_t output_index) override; - - std::vector> create_input_tensor(size_t input_index, - size_t pipeline_depth) override; - - std::vector> create_output_tensor(size_t output_index, - size_t pipeline_depth) override; + std::vector create_output_tensor(size_t output_index, size_t pipeline_depth) override; protected: - std::shared_ptr get_parameter(size_t index) const; - std::shared_ptr get_result(size_t index) const; + std::shared_ptr get_parameter(size_t index) const; + std::shared_ptr get_result(size_t index) const; bool evaluate_node(const std::shared_ptr& node, - const HostTensorVector& outputs, - const HostTensorVector& inputs) const; + ov::TensorVector& outputs, + const ov::TensorVector& inputs) const; bool m_is_compiled = false; - bool m_nan_check_enabled = false; - bool m_performance_counters_enabled = false; - std::shared_ptr m_function; - NGRAPH_SUPPRESS_DEPRECATED_START - std::unordered_map, stopwatch> m_timer_map; - NGRAPH_SUPPRESS_DEPRECATED_END + std::shared_ptr m_model; std::vector> m_nodes; - static void perform_nan_check(const std::vector>&, const Node* op = nullptr); struct InfoForNMS5 { int64_t max_output_boxes_per_class; float iou_threshold; @@ -84,6 +63,10 @@ protected: ngraph::element::Type output_type; }; - InfoForNMS5 get_info_for_nms5_eval(const op::v5::NonMaxSuppression* nms5, + InfoForNMS5 get_info_for_nms5_eval(const ov::op::v5::NonMaxSuppression* nms5, const std::vector>& inputs); }; + +} // namespace interpreter +} // namespace runtime +} // namespace ov diff --git a/src/plugins/template/backend/pass/dyn_elimination.cpp b/src/plugins/template/backend/pass/dyn_elimination.cpp deleted file mode 100644 index 51e57d60449..00000000000 --- a/src/plugins/template/backend/pass/dyn_elimination.cpp +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "dyn_elimination.hpp" - -#include - -#include "ngraph/builder/reshape.hpp" -#include "ngraph/op/broadcast.hpp" -#include "ngraph/op/range.hpp" -#include "ngraph/op/transpose.hpp" -#include "ngraph/pattern/matcher.hpp" -#include "ngraph/pattern/op/label.hpp" -#include "ngraph/runtime/reference/range.hpp" -#include "ngraph/slice_plan.hpp" - -NGRAPH_SUPPRESS_DEPRECATED_START - -using namespace std; -using namespace ngraph; - -pass::DynElimination::DynElimination() : GraphRewrite() { - construct_range(); -} - -template -std::shared_ptr make_range_replacement(const element::Type& et, - const Shape& shape, - const std::shared_ptr& start_arg, - const std::shared_ptr& step_arg) { - std::vector elements(shape_size(shape)); - std::vector start_vec = start_arg->get_vector(); - std::vector step_vec = step_arg->get_vector(); - - NGRAPH_CHECK(start_vec.size() == 1 && step_vec.size() == 1); - - runtime::reference::range(start_vec.data(), step_vec.data(), shape_size(shape), elements.data()); - - return make_shared(et, shape, elements); -} - -void pass::DynElimination::construct_range() { - auto start_arg_label = make_shared(element::f32, Shape{}, pattern::has_class()); - auto stop_arg_label = make_shared(element::f32, Shape{}, pattern::has_class()); - auto step_arg_label = make_shared(element::f32, Shape{}, pattern::has_class()); - - auto range_pat = make_shared(start_arg_label, stop_arg_label, step_arg_label); - - auto range_callback = [start_arg_label, stop_arg_label, step_arg_label](pattern::Matcher& m) { - auto pattern_map = m.get_pattern_map(); - - auto start_arg = static_pointer_cast(pattern_map[start_arg_label]); - auto step_arg = static_pointer_cast(pattern_map[step_arg_label]); - auto range_node = static_pointer_cast(m.get_match_root()); - - NGRAPH_CHECK(start_arg->get_output_partial_shape(0).rank().compatible(0) && - step_arg->get_output_partial_shape(0).rank().compatible(0)); - - auto et = range_node->get_output_element_type(0); - auto shape = range_node->get_output_shape(0); - - std::shared_ptr replacement; - -#if defined(__GNUC__) && !(__GNUC__ == 4 && __GNUC_MINOR__ == 8) -# pragma GCC diagnostic push -# pragma GCC diagnostic error "-Wswitch" -# pragma GCC diagnostic error "-Wswitch-enum" -#endif - switch (et) { - case element::Type_t::bf16: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::f16: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::f32: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::f64: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::i8: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::i16: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::i32: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::i64: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::u8: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::u16: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::u32: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::u64: - replacement = make_range_replacement(et, shape, start_arg, step_arg); - break; - case element::Type_t::i4: - case element::Type_t::u4: - case element::Type_t::u1: - case element::Type_t::undefined: - case element::Type_t::dynamic: - case element::Type_t::boolean: - NGRAPH_CHECK(false, "Internal nGraph error: unsupported element type: ", et); - break; - } -#if defined(__GNUC__) && !(__GNUC__ == 4 && __GNUC_MINOR__ == 8) -# pragma GCC diagnostic pop -#endif - - replace_node(range_node, replacement); - return true; - }; - - auto range_matcher = make_shared(range_pat, "DynElimination.Range"); - NGRAPH_SUPPRESS_DEPRECATED_START - add_matcher(range_matcher, range_callback, all_pass_property_off); - NGRAPH_SUPPRESS_DEPRECATED_END -} diff --git a/src/plugins/template/backend/pass/dyn_elimination.hpp b/src/plugins/template/backend/pass/dyn_elimination.hpp deleted file mode 100644 index 5681fd37dae..00000000000 --- a/src/plugins/template/backend/pass/dyn_elimination.hpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "ngraph/pass/graph_rewrite.hpp" -#include "ngraph/util.hpp" - -namespace ngraph { -namespace pass { -/// \brief The DynElimination pass finds dynamic operations in a graph whose -/// shape relevant inputs have already been resolved to static values, and -/// replaces those dynamic operations with the equivalent operations using -/// static inputs and attributes. -/// \details This pass should be executed after the ConstantFolding pass. -/// -/// The ConstantFolding and DynElimination passes are used together to transform -/// dynamic operations in a computation graph to static operations when the -/// graph is executed with input data. -/// -/// In the example shown below, the original graph is constructed with dynamic -/// broadcast operation. When the graph is executed with input data, the input -/// shapes become available, by applying the ConstantFolding and DynElimination -/// pass, the graph is updated with dynamic broadcast being replaced by a static -/// broadcast operation. -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -///
OriginalAfter %ConstantFoldingAfter %DynElimination
\image html dyn_broadcast_pre_constfld.svg \image html dyn_broadcast_post_constfld.svg \image html dyn_broadcast_post_dyneliminate.svg
-class DynElimination : public GraphRewrite { -public: - DynElimination(); - -private: - void construct_range(); -}; -} // namespace pass -} // namespace ngraph diff --git a/src/plugins/template/backend/pass/shape_relevance.cpp b/src/plugins/template/backend/pass/shape_relevance.cpp deleted file mode 100644 index 8aa6683adf6..00000000000 --- a/src/plugins/template/backend/pass/shape_relevance.cpp +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "pass/shape_relevance.hpp" - -#include "ngraph/graph_util.hpp" -#include "ngraph/op/constant.hpp" -#include "ngraph/op/util/op_types.hpp" - -using namespace ngraph; - -// -// This pass refreshes the "is_relevant_to_shape" flag on each parameter. A parameter will be -// flagged as relevant to shapes if there is any path from that parameter to a shape-relevant -// input that does _not_ pass through a value-irrelevant input. For example: -// -// N0[Parameter] N1[Parameter] -// | | -// | | -// | | -// N2[v1::Reshape] -// -// N1 (but not N0) will be flagged as shape-relevant, because N1 feeds into the "shape" input -// of N2. -// -// N0[Parameter] N1[Parameter] -// | | -// | N2[ShapeOf] -// | | -// N3[v1::Reshape] -// -// Neither N0 nor N1 will be flagged as shape-relevant. (N1 does feed into the "shape" input of N3, -// but only via the value-irrelevant input of ShapeOf.) -// -bool pass::ShapeRelevance::run_on_model(const std::shared_ptr& f) { - // TODO(amprocte): We are probably reinventing the wheel with the graph traversal here; the - // reason is that we need to cut the traversal short in cases where input values are - // irrelevant. See if there is a way to reduce this duplication. - - // Set of nodes that must be evaluated to determine the value of shape-relevant inputs. - std::set shape_determinants; - - // Step 1: Find root nodes (these are nodes with an output connected to a shape-relevant - // input). - for (auto& n : f->get_ops()) { - for (auto& output : n->outputs()) { - for (auto& input : output.get_target_inputs()) { - if (input.get_is_relevant_to_shapes()) { - shape_determinants.insert(n.get()); - break; - } - } - } - } - - // Step 2: Find all shape determinants. This is the transitive closure of R, where n1 R n2 - // iff there is a data flow edge from n2 to n1 and that data flow edge is not - // value-irrelevant. - bool changes_made = false; - - { - std::list to_visit{shape_determinants.begin(), shape_determinants.end()}; - std::set already_visited; - - while (!to_visit.empty()) { - auto node = to_visit.front(); - to_visit.pop_front(); - - if (already_visited.count(node) > 0) { - continue; - } - - shape_determinants.insert(node); - already_visited.insert(node); - - if (op::is_parameter(node)) { - auto node_as_param = static_cast(node); - if (!node_as_param->is_relevant_to_shapes()) { - node_as_param->set_is_relevant_to_shapes(true); - changes_made = true; - } - } - - for (size_t i = 0; i < node->get_input_size(); i++) { - if (!node->input(i).get_is_relevant_to_values()) { - continue; - } - auto source_node = node->get_input_node_ptr(i); - if (already_visited.count(source_node) == 0) { - to_visit.push_front(source_node); - } - } - } - } - - return changes_made; -} diff --git a/src/plugins/template/backend/pass/shape_relevance.hpp b/src/plugins/template/backend/pass/shape_relevance.hpp deleted file mode 100644 index 5ab0ab02292..00000000000 --- a/src/plugins/template/backend/pass/shape_relevance.hpp +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "ngraph/pass/pass.hpp" - -namespace ngraph { -namespace pass { -class ShapeRelevance : public FunctionPass { -public: - ShapeRelevance() : FunctionPass() {} - bool run_on_model(const std::shared_ptr& m) override; -}; -} // namespace pass -} // namespace ngraph diff --git a/src/plugins/template/backend/performance_counter.hpp b/src/plugins/template/backend/performance_counter.hpp deleted file mode 100644 index 5c9e49ca57f..00000000000 --- a/src/plugins/template/backend/performance_counter.hpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include - -#include "ngraph/node.hpp" - -namespace ngraph { -namespace runtime { -class PerformanceCounter { -public: - PerformanceCounter(const std::shared_ptr& n, size_t us, size_t calls) - : m_node(n), - m_total_microseconds(us), - m_call_count(calls) {} - std::shared_ptr get_node() const { - return m_node; - } - size_t total_microseconds() const { - return m_total_microseconds; - } - size_t microseconds() const { - return m_call_count == 0 ? 0 : m_total_microseconds / m_call_count; - } - size_t call_count() const { - return m_call_count; - } - std::shared_ptr m_node; - size_t m_total_microseconds; - size_t m_call_count; -}; -} // namespace runtime -} // namespace ngraph diff --git a/src/plugins/template/src/plugin.cpp b/src/plugins/template/src/plugin.cpp index 3b2765172ed..d2e6dda5ad6 100644 --- a/src/plugins/template/src/plugin.cpp +++ b/src/plugins/template/src/plugin.cpp @@ -30,7 +30,7 @@ ov::template_plugin::Plugin::Plugin() { set_device_name("TEMPLATE"); // create ngraph backend which performs inference using ngraph reference implementations - _backend = ngraph::runtime::Backend::create(); + _backend = ov::runtime::Backend::create(); // create default stream executor with a given name _waitExecutor = get_executor_manager()->get_idle_cpu_streams_executor({wait_executor_name}); diff --git a/src/plugins/template/src/plugin.hpp b/src/plugins/template/src/plugin.hpp index 0c04798e3a1..b420d5f52b3 100644 --- a/src/plugins/template/src/plugin.hpp +++ b/src/plugins/template/src/plugin.hpp @@ -50,7 +50,7 @@ private: friend class CompiledModel; friend class InferRequest; - std::shared_ptr _backend; + std::shared_ptr _backend; Configuration _cfg; std::shared_ptr _waitExecutor; }; diff --git a/src/plugins/template/src/sync_infer_request.cpp b/src/plugins/template/src/sync_infer_request.cpp index b6f2a6edf2c..ad951520432 100644 --- a/src/plugins/template/src/sync_infer_request.cpp +++ b/src/plugins/template/src/sync_infer_request.cpp @@ -131,23 +131,7 @@ void ov::template_plugin::InferRequest::infer_preprocess() { get_template_model()->get_template_plugin()->_backend->create_tensor(tensor.get_element_type(), tensor.get_shape()); auto* src_data = static_cast(tensor.data()); - auto dst_tensor = std::dynamic_pointer_cast(m_backend_input_tensors[i]); - OPENVINO_ASSERT(dst_tensor, "Template plugin error: Can't cast created tensor to HostTensor"); - auto* dst_data = dst_tensor->get_data_ptr(); - std::vector indexes(shape.size()); - for (size_t dst_idx = 0; dst_idx < ov::shape_size(shape); dst_idx++) { - size_t val = dst_idx; - size_t src_idx = 0; - for (size_t j1 = 0; j1 < indexes.size(); j1++) { - size_t j = indexes.size() - j1 - 1; - indexes[j] = val % shape[j]; - val /= shape[j]; - src_idx += indexes[j] * tensor.get_strides()[j]; - } - memcpy(dst_data + dst_idx * tensor.get_element_type().size(), - src_data + src_idx, - tensor.get_element_type().size()); - } + tensor.copy_to(m_backend_input_tensors[i]); } } // Tensors can be dynamic, so in this case we need to allocate tensors with right shape @@ -196,9 +180,8 @@ void ov::template_plugin::InferRequest::infer_postprocess() { auto host_tensor = m_backend_output_tensors[i]; ov::Output output{result->output(0).get_node(), result->output(0).get_index()}; allocate_tensor(output, [host_tensor](ov::Tensor& tensor) { - allocate_tensor_impl(tensor, host_tensor->get_element_type(), host_tensor->get_shape()); - // tensor.set_shape(host_tensor->get_shape()); - host_tensor->read(static_cast(tensor.data()), host_tensor->get_size_in_bytes()); + allocate_tensor_impl(tensor, host_tensor.get_element_type(), host_tensor.get_shape()); + host_tensor.copy_to(tensor); }); } } diff --git a/src/plugins/template/src/sync_infer_request.hpp b/src/plugins/template/src/sync_infer_request.hpp index 49b9bdaa790..1b505740c10 100644 --- a/src/plugins/template/src/sync_infer_request.hpp +++ b/src/plugins/template/src/sync_infer_request.hpp @@ -49,9 +49,9 @@ private: // for performance counters std::array, numOfStages> m_durations; - std::vector> m_backend_input_tensors; - std::vector> m_backend_output_tensors; - std::shared_ptr m_executable; + std::vector m_backend_input_tensors; + std::vector m_backend_output_tensors; + std::shared_ptr m_executable; }; // ! [infer_request:header] diff --git a/src/tests/ngraph_helpers/ngraph_functions/src/utils/ngraph_helpers.cpp b/src/tests/ngraph_helpers/ngraph_functions/src/utils/ngraph_helpers.cpp index dd3f7ed7d19..c6a7caefeb5 100644 --- a/src/tests/ngraph_helpers/ngraph_functions/src/utils/ngraph_helpers.cpp +++ b/src/tests/ngraph_helpers/ngraph_functions/src/utils/ngraph_helpers.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include "openvino/runtime/tensor.hpp" namespace ngraph { namespace helpers { @@ -84,7 +86,7 @@ std::vector>> interpreterFunction(const std::shared_ptr &function, const std::vector> &inputs, const std::vector &inputTypes) { - auto backend = runtime::Backend::create(); + auto backend = ov::runtime::Backend::create(); const auto ¶meters = function->get_parameters(); const auto ¶metersNumber = parameters.size(); @@ -98,7 +100,7 @@ std::vector>> inputTypes.size(), " types"); } - auto inputTensors = std::vector>{}; + ov::TensorVector inputTensors(parametersNumber); for (size_t i = 0; i < parametersNumber; ++i) { const auto ¶meter = parameters[i]; const auto ¶meterIndex = function->get_parameter_index(parameter); @@ -120,14 +122,14 @@ std::vector>> " has ", inputSize, " bytes"); auto tensor = backend->create_tensor(parameterType, parameterShape); - tensor->write(input.data(), parameterSize); - inputTensors.push_back(tensor); + std::memcpy(tensor.data(), input.data(), parameterSize); + inputTensors[i] = tensor; } - auto outputTensors = std::vector>{}; const auto &results = function->get_results(); + ov::TensorVector outputTensors(results.size()); for (size_t i = 0; i < results.size(); ++i) { - outputTensors.push_back(std::make_shared()); + outputTensors[i] = ov::Tensor(results[i]->get_element_type(), {0}); } auto handle = backend->compile(function); @@ -137,16 +139,16 @@ std::vector>> auto& output = outputs[resultIndex]; output.first = results[resultIndex]->get_element_type(); const auto& outputTensor = outputTensors[resultIndex]; - output.second.resize((shape_size(outputTensor->get_shape()) * outputTensor->get_element_type().bitwidth() + 7) >> 3); - outputTensors[resultIndex]->read(output.second.data(), output.second.size()); + output.second.resize((shape_size(outputTensor.get_shape()) * outputTensor.get_element_type().bitwidth() + 7) >> 3); + std::memcpy(output.second.data(), outputTensors[resultIndex].data(), output.second.size()); } return outputs; } std::vector interpretFunction(const std::shared_ptr &function, - const std::map, ov::Tensor>& inputs) { - auto backend = runtime::Backend::create(); + const std::map, ov::Tensor>& inputs) { + auto backend = ov::runtime::Backend::create(); const auto &funcInputs = function->inputs(); const auto &funcInputsNumber = funcInputs.size(); @@ -155,7 +157,7 @@ std::vector interpretFunction(const std::shared_ptr &funct "Got function (", function->get_friendly_name(), ") with ", funcInputsNumber, " parameters, but ", inputsNumber, " input blobs"); - auto inputTensors = std::vector>{}; + ov::TensorVector inputTensors(funcInputsNumber); for (size_t i = 0; i < funcInputsNumber; ++i) { const auto &input = funcInputs[i]; const auto &inputShape = input.get_shape(); @@ -178,26 +180,20 @@ std::vector interpretFunction(const std::shared_ptr &funct " has ", inputTensorSize, " bytes"); auto tensor = backend->create_tensor(inputType, inputShape); - tensor->write(inputTensor.data(), inputSize); - inputTensors.push_back(tensor); + inputTensor.copy_to(tensor); + inputTensors[i] = tensor; } - std::vector> outputTensors; const auto &results = function->get_results(); + ov::TensorVector outputTensors(results.size()); for (size_t i = 0; i < results.size(); ++i) { - outputTensors.push_back(std::make_shared()); + outputTensors[i] = ov::Tensor(results[i]->get_element_type(), {0}); } auto handle = backend->compile(function); handle->call_with_validate(outputTensors, inputTensors); - std::vector outputs; - for (const auto& outTensor : outputTensors) { - ov::Tensor tmpBuffer(outTensor->get_element_type(), outTensor->get_shape()); - outTensor->read(tmpBuffer.data(), tmpBuffer.get_byte_size()); - outputs.push_back(tmpBuffer); - } - return outputs; + return outputTensors; } std::shared_ptr foldFunction(const std::shared_ptr &function,