IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Moved template backend to new API (#15878)

Browse Source 
* Moved template backend to new API

* Fixed compilation

* Fixed some comments

* Fixed ov_core_unit_tests

* Fixed some tests

* Fixed ONNX Frontend tests

* Fixed transformation tests

* Fixed dynamic tests

* Fixed sporadic in CPU tests

* Added WA for plugin

* Fixed copy_to for scalar tensors
This commit is contained in:
Ilya Churaev 2023-03-01 07:12:33 +04:00 committed by GitHub
parent 87e714eb5c
commit e534efd4a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 331 additions and 1071 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/core/src/node.cpp
View File

@ -730,7 +730,8 @@ bool ov::Node::evaluate(ov::TensorVector& output_values, const ov::TensorVector&
OPENVINO_SUPPRESS_DEPRECATED_START
bool sts = evaluate(output, input);
OPENVINO_SUPPRESS_DEPRECATED_END
update_output_tensors(output_values, output);
if (sts)
update_output_tensors(output_values, output);
return sts;
}
@ -743,7 +744,8 @@ bool ov::Node::evaluate(ov::TensorVector& output_values,
OPENVINO_SUPPRESS_DEPRECATED_START
bool sts = evaluate(output, input, evaluationContext);
OPENVINO_SUPPRESS_DEPRECATED_END
update_output_tensors(output_values, output);
if (sts)
update_output_tensors(output_values, output);
// Call evaluate for ov::Tensor if op doesn't have evaluate with EvaluationContext
return sts ? sts : evaluate(output_values, input_values);
}

12
src/core/src/runtime/ov_tensor.cpp
View File

@ -128,6 +128,13 @@ Shape Tensor::get_shape() const {
}
void Tensor::copy_to(ov::Tensor& dst) const {
const auto& is_scalar = [](const ov::Shape& shape) {
return shape.empty() || (shape.size() == 1 && shape[0] == 1);
};
const auto shapes_equal = [is_scalar](const ov::Shape& src, const ov::Shape& dst) {
// WA for scalar tensors to copy {1} to {} or otherwise
return src == dst || (is_scalar(src) && is_scalar(dst));
};
OV_TENSOR_STATEMENT({
OPENVINO_ASSERT(dst, "Destination tensor was not initialized.");
OPENVINO_ASSERT(!is<ov::RemoteTensor>(), "Default copy to doesn't support copy from remote tensor.");
@ -140,7 +147,7 @@ void Tensor::copy_to(ov::Tensor& dst) const {
")");
if (dst.get_shape() == ov::Shape{0})
dst.set_shape(get_shape());
OPENVINO_ASSERT(dst.get_shape() == get_shape(),
OPENVINO_ASSERT(shapes_equal(get_shape(), dst.get_shape()),
"Tensor shapes are not equal. (src: ",
get_shape(),
" != dst: ",
@ -154,7 +161,8 @@ void Tensor::copy_to(ov::Tensor& dst) const {
ov::Shape cur_pos{0};
ov::Shape max_pos{1};
if (get_element_type().bitwidth() < 8 || (get_strides() == dst.get_strides() && is_continuous())) {
if (get_element_type().bitwidth() < 8 || (get_strides() == dst.get_strides() && is_continuous()) ||
(is_scalar(get_shape()) && is_scalar(dst.get_shape()))) {
// OpenVINO doesn't support strides for LP types
// or both tensors have default strides
// Strides and positions already initialized

20
src/core/tests/ov_tensor_test.cpp
View File

@ -323,8 +323,10 @@ std::vector<T> fill_data(const ov::Tensor& tensor) {
const T* data = tensor.data<T>();
auto strides = tensor.get_strides();
for (auto&& c : ngraph::CoordinateTransformBasic{tensor.get_shape()}) {
actual.emplace_back(
*(data + (c[2] * strides[2] + c[1] * strides[1] + c[0] * strides[0]) / tensor.get_element_type().size()));
size_t offset = 0;
for (size_t i = 0; i < strides.size(); i++)
offset += c[i] * strides[i];
actual.emplace_back(*(data + offset / tensor.get_element_type().size()));
}
return actual;
};
@ -394,7 +396,7 @@ void init_tensor(const ov::Tensor& tensor, bool input) {
void compare_tensors(const ov::Tensor& src, const ov::Tensor& dst) {
ASSERT_EQ(src.get_byte_size(), dst.get_byte_size());
ASSERT_EQ(src.get_shape(), dst.get_shape());
ASSERT_EQ(src.get_size(), dst.get_size());
ASSERT_EQ(src.get_element_type(), dst.get_element_type());
switch (src.get_element_type()) {
case ov::element::bf16:
@ -501,6 +503,18 @@ INSTANTIATE_TEST_SUITE_P(copy_tests,
TestParams {
ov::Shape{3, 2, 2}, ov::Strides{64, 16, 8},
ov::Shape{3, 2, 2}, ov::Strides{128, 24, 8}
},
TestParams {
ov::Shape{}, {},
{}, {}
},
TestParams {
ov::Shape{1}, {},
{}, {}
},
TestParams {
ov::Shape{}, {},
{1}, {}
}
)));
// clang-format on

11
src/plugins/template/backend/CMakeLists.txt
View File

@ -5,16 +5,11 @@
set (SRC
backend.cpp
backend.hpp
cache.cpp
cache.hpp
executable.cpp
executable.hpp
performance_counter.hpp
pass/dyn_elimination.cpp
pass/dyn_elimination.hpp
pass/shape_relevance.cpp
pass/shape_relevance.hpp
int_backend.cpp int_executable.cpp evaluates_map.cpp
int_backend.cpp
int_executable.cpp
evaluates_map.cpp
)
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")

31
src/plugins/template/backend/backend.cpp
View File

@ -9,36 +9,13 @@
#include "int_backend.hpp"
#include "ngraph/file_util.hpp"
#include "ngraph/util.hpp"
#include "openvino/core/except.hpp"
#include "openvino/util/file_util.hpp"
using namespace std;
using namespace ngraph;
ov::runtime::Backend::~Backend() = default;
runtime::Backend::~Backend() {}
std::shared_ptr<runtime::Backend> runtime::Backend::create() {
auto inner_backend = make_shared<interpreter::INTBackend>();
std::shared_ptr<ov::runtime::Backend> ov::runtime::Backend::create() {
auto inner_backend = std::make_shared<interpreter::INTBackend>();
return inner_backend;
}
std::shared_ptr<ngraph::runtime::Tensor> runtime::Backend::create_dynamic_tensor(
const ngraph::element::Type& /* element_type */,
const PartialShape& /* shape */) {
throw std::invalid_argument("This backend does not support dynamic tensors");
}
bool runtime::Backend::is_supported(const Node& /* node */) const {
// The default behavior is that a backend does not support any ops. If this is not the case
// then override this method and enhance.
return false;
}
std::shared_ptr<runtime::Executable> runtime::Backend::load(istream& /* input_stream */) {
throw runtime_error("load operation unimplemented.");
}
bool runtime::Backend::set_config(const map<string, string>& /* config */, string& error) {
error = "set_config not supported";
return false;
}

72
src/plugins/template/backend/backend.hpp
View File

@ -8,23 +8,16 @@
#include <mutex>
#include "executable.hpp"
#include "ngraph/function.hpp"
#include "ngraph/shape.hpp"
#include "ngraph/type/element_type.hpp"
#include "ngraph/util.hpp"
#include "performance_counter.hpp"
#include "openvino/core/model.hpp"
#include "openvino/core/type/element_type.hpp"
namespace ngraph {
namespace ov {
namespace runtime {
class Tensor;
class Backend;
} // namespace runtime
} // namespace ngraph
/// \brief Interface to a generic backend.
///
/// Backends are responsible for function execution and value allocation.
class ngraph::runtime::Backend {
class Backend {
public:
virtual ~Backend();
/// \brief Create a new Backend object
@ -44,14 +37,13 @@ public:
/// passed as an output to a function the tensor will have a type and shape after executing
/// a call.
/// \returns shared_ptr to a new backend-specific tensor
virtual std::shared_ptr<ngraph::runtime::Tensor> create_tensor() = 0;
virtual ov::Tensor create_tensor() = 0;
/// \brief Create a tensor specific to this backend
/// \param element_type The type of the tensor element
/// \param shape The shape of the tensor
/// \returns shared_ptr to a new backend-specific tensor
virtual std::shared_ptr<ngraph::runtime::Tensor> create_tensor(const ngraph::element::Type& element_type,
const Shape& shape) = 0;
virtual ov::Tensor create_tensor(const ov::element::Type& element_type, const Shape& shape) = 0;
/// \brief Create a tensor specific to this backend
/// \param element_type The type of the tensor element
@ -60,59 +52,23 @@ public:
/// must be sufficient to contain the tensor. The lifetime of the buffer is the
/// responsibility of the caller.
/// \returns shared_ptr to a new backend-specific tensor
virtual std::shared_ptr<ngraph::runtime::Tensor> create_tensor(const ngraph::element::Type& element_type,
const Shape& shape,
void* memory_pointer) = 0;
virtual ov::Tensor create_tensor(const ov::element::Type& element_type,
const Shape& shape,
void* memory_pointer) = 0;
/// \brief Create a tensor of C type T specific to this backend
/// \param shape The shape of the tensor
/// \returns shared_ptr to a new backend specific tensor
template <typename T>
std::shared_ptr<ngraph::runtime::Tensor> create_tensor(const Shape& shape) {
ov::Tensor create_tensor(const Shape& shape) {
return create_tensor(element::from<T>(), shape);
}
/// \brief Create a dynamic tensor specific to this backend, if the backend supports dynamic
/// tensors.
/// \param element_type The type of the tensor element
/// \param shape The shape of the tensor
/// \returns shared_ptr to a new backend-specific tensor
/// \throws std::invalid_argument if the backend does not support dynamic tensors
virtual std::shared_ptr<ngraph::runtime::Tensor> create_dynamic_tensor(const ngraph::element::Type& element_type,
const PartialShape& shape);
/// \returns `true` if this backend supports dynamic tensors, else `false`.
virtual bool supports_dynamic_tensors() {
return false;
}
/// \brief Compiles a Function.
/// \param func The function to compile
/// \returns compiled function or nullptr on failure
virtual std::shared_ptr<Executable> compile(std::shared_ptr<Function> func,
bool enable_performance_data = false) = 0;
/// \brief Loads a previously saved Executable object from a stream.
/// \param input_stream the opened input stream containing the saved Executable
/// \returns A compiled function or throws an exception on error
virtual std::shared_ptr<Executable> load(std::istream& input_stream);
/// \brief Test if a backend is capable of supporting an op
/// \param node is the op to test.
/// \returns true if the op is supported, false otherwise.
virtual bool is_supported(const Node& node) const;
/// \brief Allows sending backend specific configuration. The map contains key, value pairs
/// specific to a particluar backend. The definition of these key, value pairs is
/// defined by each backend.
/// \param config The configuration map sent to the backend
/// \param error An error string describing any error encountered
/// \returns true if the configuration is supported, false otherwise. On false the error
/// parameter value is valid.
virtual bool set_config(const std::map<std::string, std::string>& config, std::string& error);
/// \brief Get the version of the backend
/// The default value of 0.0.0 is chosen to be a parsable version number
virtual std::string get_version() const {
return "0.0.0";
}
virtual std::shared_ptr<Executable> compile(std::shared_ptr<ov::Model> model) = 0;
};
} // namespace runtime
} // namespace ov

95
src/plugins/template/backend/cache.cpp
View File

@ -1,95 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "cache.hpp"
#include "ngraph/env_util.hpp"
using namespace ngraph;
using namespace std;
// Constructor
runtime::LRUCache::LRUCache() {
m_cache_size = 1024;
m_map = {};
m_list = {};
}
// Destructor
runtime::LRUCache::~LRUCache() {
m_list.clear();
m_map.clear();
m_clone_function_map.clear();
}
void runtime::LRUCache::convert_shape_to_string(const vector<int>& shape, ostringstream& key) {
if (!shape.empty()) {
std::copy(shape.begin(), shape.end(), std::ostream_iterator<int>(key, ", "));
}
}
void runtime::LRUCache::add_entry(const vector<int>& shape,
shared_ptr<runtime::Executable> exec,
shared_ptr<Function> func) {
std::lock_guard<std::mutex> guard(m_mutex);
ostringstream key;
// check if the list is empty
if (m_list.size() == static_cast<size_t>(m_cache_size)) {
ostringstream key;
convert_shape_to_string(m_list.back(), key);
m_list.pop_back();
m_map.erase(key.str());
}
convert_shape_to_string(shape, key);
m_map.insert({key.str(), exec});
m_list.push_front(shape);
m_clone_function_map.insert({key.str(), func});
}
bool runtime::LRUCache::is_cached(const vector<int>& shape) {
for (auto itr = m_list.begin(); itr != m_list.end(); itr++) {
if (*itr == shape) {
return true;
}
}
return false;
}
shared_ptr<runtime::Executable> runtime::LRUCache::get_cached_entry(const vector<int>& shape) {
std::lock_guard<std::mutex> guard(m_mutex);
ostringstream key;
convert_shape_to_string(shape, key);
// find the entry and return the function
auto it = m_map.find(key.str());
if (it == m_map.end()) {
throw ngraph_error("Entry not found in cache");
} else {
// update list to push this reference to the front
for (auto itr = m_list.begin(); itr != m_list.end(); itr++) {
if (*itr == shape) {
m_list.remove(shape);
m_list.push_front(shape);
break;
}
}
return it->second;
}
}
// Need the clone function to get the output shape so that
// storage can be allocated for output
shared_ptr<Function> runtime::LRUCache::get_cloned_function(const vector<int>& shape) {
std::lock_guard<std::mutex> guard(m_mutex);
ostringstream key;
convert_shape_to_string(shape, key);
// find the entry and return the function
auto it = m_clone_function_map.find(key.str());
if (it == m_clone_function_map.end()) {
throw ngraph_error("Cloned function not found");
}
return it->second;
}

45
src/plugins/template/backend/cache.hpp
View File

@ -1,45 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <algorithm>
#include <iostream>
#include <iterator>
#include <list>
#include <mutex>
#include <sstream>
#include <string>
#include <unordered_map>
#include "executable.hpp"
#include "ngraph/function.hpp"
#include "ngraph/shape.hpp"
namespace ngraph {
namespace runtime {
class LRUCache : public std::enable_shared_from_this<LRUCache> {
public:
using GraphCache = std::unordered_map<std::string, std::shared_ptr<Executable>>;
using ClonedFunctionMap = std::unordered_map<std::string, std::shared_ptr<Function>>;
LRUCache();
virtual ~LRUCache();
void add_entry(const std::vector<int>& shape, std::shared_ptr<Executable> exec, std::shared_ptr<Function> func);
bool is_cached(const std::vector<int>& shape);
std::shared_ptr<Executable> get_cached_entry(const std::vector<int>& shape);
void convert_shape_to_string(const std::vector<int>& shape, std::ostringstream& key);
std::shared_ptr<Function> get_cloned_function(const std::vector<int>& shape);
private:
int m_cache_size;
GraphCache m_map;
ClonedFunctionMap m_clone_function_map;
std::list<std::vector<int>> m_list;
std::mutex m_mutex;
};
} // namespace runtime
} // namespace ngraph

44
src/plugins/template/backend/evaluates_map.cpp
View File

@ -95,6 +95,7 @@
#include "ngraph/runtime/reference/convert_color_nv12.hpp"
#include "ov_ops/augru_cell.hpp"
#include "ov_ops/augru_sequence.hpp"
#include "tensor_conversion_util.hpp"
using namespace ngraph;
using namespace std;
@ -3302,46 +3303,21 @@ runtime::reference::custom_evaluate_function evaluate = [](const std::shared_ptr
inputsNumber,
" input blobs");
auto inputTensors = std::vector<std::shared_ptr<runtime::Tensor>>{};
for (const auto& parameter : parameters) {
const auto& parameterIndex = function->get_parameter_index(parameter);
const auto& parameterShape = parameter->get_shape();
const auto& parameterType = parameter->get_element_type();
const auto& parameterSize = shape_size(parameterShape) * parameterType.size();
const auto& input = inputs[parameterIndex];
const auto& inputSize = input->get_size_in_bytes();
NGRAPH_CHECK(parameterSize == inputSize,
"Got parameter (",
parameter->get_friendly_name(),
") of size ",
parameterSize,
" bytes, but corresponding input with index ",
parameterIndex,
" has ",
inputSize,
" bytes");
auto tensor = std::make_shared<runtime::HostTensor>(parameterType, parameterShape);
tensor->write(input->get_data_ptr(), parameterSize);
inputTensors.push_back(tensor);
}
const auto& results = function->get_results();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> outputTensors;
outputTensors.reserve(results.size());
outputs.reserve(results.size());
for (size_t i = 0; i < results.size(); ++i) {
outputTensors.push_back(std::make_shared<HostTensor>());
outputs.push_back(std::make_shared<HostTensor>(results[i]->output(0)));
}
auto backend = runtime::Backend::create();
auto backend = ov::runtime::Backend::create();
auto handle = backend->compile(function);
OPENVINO_SUPPRESS_DEPRECATED_START
auto outputTensors = ov::util::wrap_tensors(outputs);
auto inputTensors = ov::util::wrap_tensors(inputs);
handle->call_with_validate(outputTensors, inputTensors);
outputs.reserve(outputTensors.size());
for (const auto& tensor : outputTensors) {
auto host_tensor = static_pointer_cast<runtime::HostTensor>(tensor);
outputs.push_back(host_tensor);
}
ov::util::update_output_host_tensors(outputs, outputTensors);
OPENVINO_SUPPRESS_DEPRECATED_END
};
} // namespace ti_v0

120
src/plugins/template/backend/executable.cpp
View File

@ -6,133 +6,81 @@
#include <sstream>
#include "ngraph/file_util.hpp"
#include "ngraph/runtime/tensor.hpp"
#include "ngraph/util.hpp"
#include "openvino/core/except.hpp"
using namespace std;
using namespace ngraph;
ov::runtime::Executable::Executable() {}
runtime::Executable::Executable() {}
ov::runtime::Executable::~Executable() {}
runtime::Executable::~Executable() {}
bool runtime::Executable::call_with_validate(const vector<shared_ptr<runtime::Tensor>>& outputs,
const vector<shared_ptr<runtime::Tensor>>& inputs) {
bool ov::runtime::Executable::call_with_validate(std::vector<ov::Tensor>& outputs,
const std::vector<ov::Tensor>& inputs) {
validate(outputs, inputs);
return call(outputs, inputs);
}
void runtime::Executable::validate(const vector<std::shared_ptr<runtime::Tensor>>& outputs,
const vector<std::shared_ptr<runtime::Tensor>>& inputs) {
void ov::runtime::Executable::validate(const std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs) {
const ParameterVector& parameters = get_parameters();
const ResultVector& results = get_results();
if (parameters.size() != inputs.size()) {
stringstream ss;
std::stringstream ss;
ss << "Call input count " << inputs.size() << " does not match Function's Parameter count "
<< parameters.size();
throw runtime_error(ss.str());
throw std::runtime_error(ss.str());
}
if (results.size() != outputs.size()) {
stringstream ss;
std::stringstream ss;
ss << "Call output count " << outputs.size() << " does not match Function's Result count " << results.size();
throw runtime_error(ss.str());
throw std::runtime_error(ss.str());
}
for (size_t i = 0; i < parameters.size(); i++) {
if (parameters[i]->get_element_type().is_static() &&
parameters[i]->get_element_type() != inputs[i]->get_element_type()) {
stringstream ss;
ss << "Input " << i << " type '" << inputs[i]->get_element_type() << "' does not match Parameter type '"
parameters[i]->get_element_type() != inputs[i].get_element_type()) {
std::stringstream ss;
ss << "Input " << i << " type '" << inputs[i].get_element_type() << "' does not match Parameter type '"
<< parameters[i]->get_element_type() << "'";
throw runtime_error(ss.str());
}
if (!(parameters[i]->get_output_partial_shape(0).relaxes(inputs[i]->get_partial_shape()))) {
stringstream ss;
ss << "Input " << i << " shape " << inputs[i]->get_partial_shape() << " does not match Parameter shape "
<< parameters[i]->get_output_partial_shape(0);
throw runtime_error(ss.str());
throw std::runtime_error(ss.str());
}
}
for (size_t i = 0; i < results.size(); i++) {
if (outputs[i]->get_element_type().is_static() && results[i]->get_element_type().is_static() &&
results[i]->get_element_type() != outputs[i]->get_element_type()) {
stringstream ss;
ss << "Output " << i << " type '" << outputs[i]->get_element_type() << "' does not match Result type '"
if (outputs[i].get_element_type().is_static() && results[i]->get_element_type().is_static() &&
results[i]->get_element_type() != outputs[i].get_element_type()) {
std::stringstream ss;
ss << "Output " << i << " type '" << outputs[i].get_element_type() << "' does not match Result type '"
<< results[i]->get_element_type() << "'";
throw runtime_error(ss.str());
}
if (!outputs[i]->get_partial_shape().relaxes(results[i]->get_output_partial_shape(0))) {
stringstream ss;
ss << "Output " << i << " shape " << outputs[i]->get_partial_shape() << " does not match max Result shape "
<< results[i]->get_output_partial_shape(0).get_max_shape();
throw runtime_error(ss.str());
throw std::runtime_error(ss.str());
}
}
}
const ngraph::ParameterVector& runtime::Executable::get_parameters() const {
const ov::ParameterVector& ov::runtime::Executable::get_parameters() const {
return m_parameters;
}
const ngraph::ResultVector& runtime::Executable::get_results() const {
const ov::ResultVector& ov::runtime::Executable::get_results() const {
return m_results;
}
size_t runtime::Executable::get_preferred_pipeline_depth() const {
return 2;
void ov::runtime::Executable::set_parameters_and_results(const ov::Model& model) {
m_parameters = model.get_parameters();
m_results = model.get_results();
}
void runtime::Executable::set_parameters_and_results(const Function& func) {
m_parameters = func.get_parameters();
m_results = func.get_results();
ov::Tensor ov::runtime::Executable::create_input_tensor(size_t /* input_index */) {
OPENVINO_NOT_IMPLEMENTED;
}
vector<runtime::PerformanceCounter> runtime::Executable::get_performance_data() const {
return vector<PerformanceCounter>();
ov::Tensor ov::runtime::Executable::create_output_tensor(size_t /* output_index */) {
OPENVINO_NOT_IMPLEMENTED;
}
void runtime::Executable::save(std::ostream& /* output_stream */) {
throw runtime_error("save operation unimplemented.");
std::vector<ov::Tensor> ov::runtime::Executable::create_input_tensor(size_t /* input_index */,
size_t /* pipeline_depth */) {
OPENVINO_NOT_IMPLEMENTED;
}
shared_ptr<runtime::Tensor> runtime::Executable::create_input_tensor(size_t /* input_index */) {
throw runtime_error("create_input_tensor unimplemented");
}
shared_ptr<runtime::Tensor> runtime::Executable::create_input_tensor(size_t /* input_index */,
void* /* memory_pointer */) {
throw runtime_error("create_input_tensor unimplemented");
}
shared_ptr<runtime::Tensor> runtime::Executable::create_output_tensor(size_t /* output_index */) {
throw runtime_error("create_output_tensor unimplemented");
}
shared_ptr<runtime::Tensor> runtime::Executable::create_output_tensor(size_t /* output_index */,
void* /* memory_pointer */) {
throw runtime_error("create_output_tensor unimplemented");
}
vector<shared_ptr<runtime::Tensor>> runtime::Executable::create_input_tensor(size_t /* input_index */,
size_t /* pipeline_depth */) {
throw runtime_error("create_input_tensor unimplemented");
}
vector<shared_ptr<runtime::Tensor>> runtime::Executable::create_input_tensor(size_t /* input_index */,
size_t /* pipeline_depth */,
std::vector<void*> /* memory_pointer */) {
throw runtime_error("create_input_tensor unimplemented");
}
vector<shared_ptr<runtime::Tensor>> runtime::Executable::create_output_tensor(size_t /* output_index */,
size_t /* pipeline_depth */) {
throw runtime_error("create_output_tensor unimplemented");
}
vector<shared_ptr<runtime::Tensor>> runtime::Executable::create_output_tensor(size_t /* output_index */,
size_t /* pipeline_depth */,
std::vector<void*> /* memory_pointer */) {
throw runtime_error("create_output_tensor unimplemented");
std::vector<ov::Tensor> ov::runtime::Executable::create_output_tensor(size_t /* output_index */,
size_t /* pipeline_depth */) {
OPENVINO_NOT_IMPLEMENTED;
}

104
src/plugins/template/backend/executable.hpp
View File

@ -6,19 +6,15 @@
#include <memory>
#include "ngraph/function.hpp"
#include "ngraph/runtime/tensor.hpp"
#include "ngraph/shape.hpp"
#include "ngraph/type/element_type.hpp"
#include "performance_counter.hpp"
#include "openvino/core/model.hpp"
#include "openvino/core/node_vector.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/runtime/tensor.hpp"
namespace ngraph {
namespace ov {
namespace runtime {
class Executable;
}
} // namespace ngraph
class ngraph::runtime::Executable {
class Executable {
public:
Executable();
virtual ~Executable();
@ -26,71 +22,38 @@ public:
/// \param outputs vector of runtime::Tensor used as outputs
/// \param inputs vector of runtime::Tensor used as inputs
/// \returns true if iteration is successful, false otherwise
virtual bool call(const std::vector<std::shared_ptr<runtime::Tensor>>& outputs,
const std::vector<std::shared_ptr<runtime::Tensor>>& inputs) = 0;
virtual bool call(std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs) = 0;
/// \brief Executes a single iteration of a Function.
/// \param outputs vector of runtime::Tensor used as outputs
/// \param inputs vector of runtime::Tensor used as inputs
/// \returns true if iteration is successful, false otherwise
bool call_with_validate(const std::vector<std::shared_ptr<runtime::Tensor>>& outputs,
const std::vector<std::shared_ptr<runtime::Tensor>>& inputs);
/// \brief Collect performance information gathered on a Function.
/// \returns Vector of PerformanceCounter information.
virtual std::vector<PerformanceCounter> get_performance_data() const;
bool call_with_validate(std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs);
/// \brief Validates a Function.
/// \param outputs vector of runtime::Tensor used as outputs
/// \param inputs vector of runtime::Tensor used as inputs
void validate(const std::vector<std::shared_ptr<runtime::Tensor>>& outputs,
const std::vector<std::shared_ptr<runtime::Tensor>>& inputs);
void validate(const std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs);
/// \brief Query the input Parameters
/// \returns an ngraph::op::ParameterVector of all input parameters
const ngraph::ParameterVector& get_parameters() const;
const ov::ParameterVector& get_parameters() const;
/// \brief Query the output Results
/// \returns an ngraph::ResultVector of all input parameters
const ngraph::ResultVector& get_results() const;
/// \brief Get the preferred pipeline_depth for this executable
/// \returns preferred pipeline_depth
virtual size_t get_preferred_pipeline_depth() const;
/// \brief Save this compiled Executable to an output stream.
/// Saved stream may be read with Backend::load
virtual void save(std::ostream& output_stream);
const ov::ResultVector& get_results() const;
/// \brief Create an input Tensor
/// \param input_index The index position in the input Parameter vector. This would be the same
/// order of Parameters passed into the inputs in the call() method.
/// \returns A Tensor
virtual std::shared_ptr<runtime::Tensor> create_input_tensor(size_t input_index);
/// \brief Create an input Tensor
/// \param input_index The index position in the input Parameter vector. This would be the same
/// order of Parameters passed into the inputs in the call() method.
/// \param memory_pointer A pointer to a buffer used for this tensor. The size of the buffer
/// must be sufficient to contain the tensor. The lifetime of the buffer is the
/// responsibility of the caller and must outlive the created Tensor.
/// \returns A Tensor
virtual std::shared_ptr<runtime::Tensor> create_input_tensor(size_t input_index, void* memory_pointer);
virtual ov::Tensor create_input_tensor(size_t input_index);
/// \brief Create an output Tensor
/// \param output_index The index position in the output Result vector. This would be the same
/// order of Results passed into the outputs in the call() method.
/// \returns A Tensor
virtual std::shared_ptr<runtime::Tensor> create_output_tensor(size_t output_index);
/// \brief Create an output Tensor
/// \param output_index The index position in the output Result vector. This would be the same
/// order of Results passed into the outputs in the call() method.
/// \param memory_pointer A pointer to a buffer used for this tensor. The size of the buffer
/// must be sufficient to contain the tensor. The lifetime of the buffer is the
/// responsibility of the caller and must outlive the created Tensor.
/// \returns A Tensor
virtual std::shared_ptr<runtime::Tensor> create_output_tensor(size_t output_index, void* memory_pointer);
virtual ov::Tensor create_output_tensor(size_t output_index);
/// \brief Create a vector of input Tensors
/// \param input_index The index position in the input Parameter vector. This would be the same
@ -98,21 +61,7 @@ public:
/// \param pipeline_depth The number of stages in the input pipeline. For double-buffered input
/// you would specify pipeline_depth=2
/// \returns A vector of Tensors, one for each stage of the pipeline
virtual std::vector<std::shared_ptr<runtime::Tensor>> create_input_tensor(size_t input_index,
size_t pipeline_depth);
/// \brief Create a vector of input Tensors
/// \param input_index The index position in the input Parameter vector. This would be the same
/// order of Parameters passed into the inputs in the call() method.
/// \param pipeline_depth The number of stages in the input pipeline. For double-buffered input
/// you would specify pipeline_depth=2
/// \param memory_pointers A vector of pointers to buffers used for this tensors. The size of
/// the buffer must be sufficient to contain the tensor. The lifetime of the buffers is the
/// responsibility of the caller and must outlive the created Tensor.
/// \returns A vector of Tensors, one for each stage of the pipeline
virtual std::vector<std::shared_ptr<runtime::Tensor>> create_input_tensor(size_t input_index,
size_t pipeline_depth,
std::vector<void*> memory_pointers);
virtual std::vector<ov::Tensor> create_input_tensor(size_t input_index, size_t pipeline_depth);
/// \brief Create a vector of output Tensors
/// \param output_index The index position in the output Result vector. This would be the same
@ -120,28 +69,17 @@ public:
/// \param pipeline_depth The number of stages in the output pipeline. For double-buffered
/// output you would specify pipeline_depth=2
/// \returns A vector of Tensors, one for each stage of the pipeline
virtual std::vector<std::shared_ptr<runtime::Tensor>> create_output_tensor(size_t output_index,
size_t pipeline_depth);
/// \brief Create a vector of output Tensors
/// \param output_index The index position in the output Result vector. This would be the same
/// order of Results passed into the outputs in the call() method.
/// \param pipeline_depth The number of stages in the output pipeline. For double-buffered
/// output you would specify pipeline_depth=2
/// \param memory_pointers A vector of pointers to buffers used for this tensors. The size of
/// the buffer must be sufficient to contain the tensor. The lifetime of the buffers is the
/// responsibility of the caller and must outlive the created Tensor.
/// \returns A vector of Tensors, one for each stage of the pipeline
virtual std::vector<std::shared_ptr<runtime::Tensor>> create_output_tensor(size_t output_index,
size_t pipeline_depth,
std::vector<void*> memory_pointers);
virtual std::vector<ov::Tensor> create_output_tensor(size_t output_index, size_t pipeline_depth);
protected:
/// \brief Called at the end of compile to the values to be returned by get_parameters
/// and get_results
/// \param func The function with Results fully resolved.
void set_parameters_and_results(const Function& func);
void set_parameters_and_results(const ov::Model& model);
ngraph::ParameterVector m_parameters;
ngraph::ResultVector m_results;
ov::ParameterVector m_parameters;
ov::ResultVector m_results;
};
} // namespace runtime
} // namespace ov

53
src/plugins/template/backend/int_backend.cpp
View File

@ -5,54 +5,27 @@
#include "int_backend.hpp"
#include "int_executable.hpp"
#include "ngraph/except.hpp"
#include "ngraph/runtime/host_tensor.hpp"
#include "ngraph/util.hpp"
using namespace std;
using namespace ngraph;
ov::runtime::interpreter::INTBackend::INTBackend() {}
runtime::interpreter::INTBackend::INTBackend() {}
runtime::interpreter::INTBackend::INTBackend(const vector<string>& unsupported_op_name_list)
ov::runtime::interpreter::INTBackend::INTBackend(const std::vector<std::string>& unsupported_op_name_list)
: m_unsupported_op_name_list{unsupported_op_name_list.begin(), unsupported_op_name_list.end()} {}
shared_ptr<runtime::Tensor> runtime::interpreter::INTBackend::create_tensor() {
return make_shared<runtime::HostTensor>();
ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor() {
return ov::Tensor();
}
shared_ptr<runtime::Tensor> runtime::interpreter::INTBackend::create_tensor(const element::Type& type,
const Shape& shape) {
return make_shared<runtime::HostTensor>(type, shape);
ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor(const element::Type& type, const Shape& shape) {
return ov::Tensor(type, shape);
}
shared_ptr<runtime::Tensor> runtime::interpreter::INTBackend::create_dynamic_tensor(const element::Type& type,
const PartialShape& pshape) {
return make_shared<runtime::HostTensor>(type, pshape);
ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor(const element::Type& type,
const Shape& shape,
void* memory_pointer) {
return ov::Tensor(type, shape, memory_pointer);
}
shared_ptr<runtime::Tensor> runtime::interpreter::INTBackend::create_tensor(const element::Type& type,
const Shape& shape,
void* memory_pointer) {
return make_shared<runtime::HostTensor>(type, shape, memory_pointer);
}
shared_ptr<runtime::Executable> runtime::interpreter::INTBackend::compile(shared_ptr<Function> function,
bool enable_performance_collection) {
return make_shared<INTExecutable>(function, enable_performance_collection);
}
bool runtime::interpreter::INTBackend::is_supported(const Node& node) const {
return m_unsupported_op_name_list.find(node.description()) == m_unsupported_op_name_list.end();
}
bool runtime::interpreter::INTBackend::set_config(const map<string, string>& config, string& error) {
bool rc = false;
auto it = config.find("test_echo");
error = "";
if (it != config.end()) {
error = it->second;
rc = true;
}
return rc;
std::shared_ptr<ov::runtime::Executable> ov::runtime::interpreter::INTBackend::compile(
std::shared_ptr<ov::Model> model) {
return std::make_shared<INTExecutable>(model);
}

29
src/plugins/template/backend/int_backend.hpp
View File

@ -11,18 +11,13 @@
#include <vector>
#include "backend.hpp"
#include "ngraph/runtime/tensor.hpp"
#include "openvino/core/model.hpp"
namespace ngraph {
namespace ov {
namespace runtime {
namespace interpreter {
class INTBackend;
class INTExecutable;
} // namespace interpreter
} // namespace runtime
} // namespace ngraph
class ngraph::runtime::interpreter::INTBackend : public Backend {
class INTBackend : public Backend {
public:
INTBackend();
INTBackend(const std::vector<std::string>& unsupported_op_name_list);
@ -30,20 +25,18 @@ public:
INTBackend(INTBackend&&) = delete;
INTBackend& operator=(const INTBackend&) = delete;
std::shared_ptr<Tensor> create_tensor() override;
ov::Tensor create_tensor() override;
std::shared_ptr<Tensor> create_tensor(const element::Type& type, const Shape& shape, void* memory_pointer) override;
ov::Tensor create_tensor(const element::Type& type, const Shape& shape, void* memory_pointer) override;
std::shared_ptr<Tensor> create_tensor(const element::Type& type, const Shape& shape) override;
std::shared_ptr<Tensor> create_dynamic_tensor(const element::Type& type, const PartialShape& shape) override;
ov::Tensor create_tensor(const element::Type& type, const Shape& shape) override;
std::shared_ptr<Executable> compile(std::shared_ptr<Function> function,
bool enable_performance_data = false) override;
bool is_supported(const Node& node) const override;
bool set_config(const std::map<std::string, std::string>& config, std::string& error) override;
std::shared_ptr<Executable> compile(std::shared_ptr<ov::Model> model) override;
private:
std::set<std::string> m_unsupported_op_name_list;
};
} // namespace interpreter
} // namespace runtime
} // namespace ov

299
src/plugins/template/backend/int_executable.cpp
View File

@ -5,32 +5,85 @@
#include "int_executable.hpp"
#include <cstring>
#include <limits>
#include <openvino/op/util/variable_context.hpp>
#include "evaluates_map.hpp"
#include "ngraph/except.hpp"
#include "ngraph/ops.hpp"
#include "ngraph/type/bfloat16.hpp"
#include "ngraph/type/float16.hpp"
#include "ngraph/util.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/op/result.hpp"
#include "openvino/op/util/op_types.hpp"
#include "tensor_conversion_util.hpp"
using namespace std;
using namespace ngraph;
NGRAPH_SUPPRESS_DEPRECATED_START
class TemporaryOverrideOutputs {
std::shared_ptr<Node> node;
std::vector<PartialShape> orig_shapes;
namespace {
class DynamicTensor : public ngraph::runtime::HostTensor {
private:
ov::Tensor tensor;
public:
TemporaryOverrideOutputs(std::shared_ptr<Node> node, const std::vector<std::shared_ptr<HostTensor>>& args)
: node(node) {
DynamicTensor(const ov::element::Type& type) : ngraph::runtime::HostTensor(type, ov::PartialShape::dynamic()) {}
ov::Tensor get_tensor() {
return tensor;
}
protected:
void allocate_buffer() override {
OPENVINO_ASSERT(get_partial_shape().is_static(),
"Attempt to allocate buffer for tensor with partial shape: ",
get_partial_shape());
OPENVINO_ASSERT(get_element_type().is_static(),
"Attempt to allocate buffer for tensor with dynamic type: ",
get_element_type());
m_buffer_size = m_descriptor->size();
tensor = ov::Tensor(get_element_type(), get_partial_shape().get_shape());
m_memory_pointer = tensor.data();
m_aligned_buffer_pool = m_memory_pointer;
}
};
inline ngraph::HostTensorPtr make_tmp_host_tensor(const ov::Tensor& t) {
OPENVINO_SUPPRESS_DEPRECATED_START
if (!t) {
return std::make_shared<DynamicTensor>(ov::element::dynamic);
} else if (t.get_shape() == ov::Shape{0, std::numeric_limits<size_t>::max()}) {
return std::make_shared<DynamicTensor>(t.get_element_type());
} else {
return std::make_shared<ngraph::runtime::HostTensor>(t.get_element_type(), t.get_shape(), t.data());
}
OPENVINO_SUPPRESS_DEPRECATED_END
}
inline ngraph::HostTensorVector create_tmp_tensors(const ov::TensorVector& tensors) {
ngraph::HostTensorVector result;
result.reserve(tensors.size());
for (const auto& tensor : tensors) {
result.push_back(make_tmp_host_tensor(tensor));
}
return result;
}
inline void update_output_tensors(ov::TensorVector& output_values, const ngraph::HostTensorVector& outputs) {
OPENVINO_ASSERT(output_values.size() == outputs.size());
for (size_t i = 0; i < outputs.size(); i++) {
if (auto dyn_output = std::dynamic_pointer_cast<DynamicTensor>(outputs[i])) {
output_values[i] = dyn_output->get_tensor();
}
}
}
} // namespace
class TemporaryOverrideOutputs {
std::shared_ptr<ov::Node> node;
std::vector<ov::PartialShape> orig_shapes;
public:
TemporaryOverrideOutputs(std::shared_ptr<ov::Node> node, const std::vector<ov::Tensor>& args) : node(node) {
for (size_t i = 0; i < args.size(); ++i) {
auto output = node->get_input_source_output(i);
orig_shapes.push_back(output.get_partial_shape());
output.get_tensor().set_partial_shape(args[i]->get_shape());
output.get_tensor().set_partial_shape(args[i].get_shape());
}
}
@ -42,43 +95,23 @@ public:
}
};
runtime::interpreter::INTExecutable::INTExecutable(const shared_ptr<Function>& function,
bool enable_performance_collection)
: m_is_compiled{true},
m_performance_counters_enabled{enable_performance_collection} {
m_function = clone_function(*function);
for (auto node : m_function->get_ordered_ops()) {
ov::runtime::interpreter::INTExecutable::INTExecutable(const std::shared_ptr<ov::Model>& model) : m_is_compiled{true} {
m_model = model->clone();
for (auto node : m_model->get_ordered_ops()) {
m_nodes.push_back(node);
}
set_parameters_and_results(*m_function);
set_parameters_and_results(*m_model);
}
bool runtime::interpreter::INTExecutable::call(const vector<shared_ptr<runtime::Tensor>>& outputs,
const vector<shared_ptr<runtime::Tensor>>& inputs) {
// convert inputs to HostTensor
vector<shared_ptr<HostTensor>> func_inputs;
for (const auto& tensor : inputs) {
auto host_tensor = static_pointer_cast<runtime::HostTensor>(tensor);
func_inputs.push_back(host_tensor);
}
if (m_nan_check_enabled) {
perform_nan_check(func_inputs);
}
// convert outputs to HostTensor
vector<shared_ptr<HostTensor>> func_outputs;
for (const auto& tensor : outputs) {
auto host_tensor = static_pointer_cast<runtime::HostTensor>(tensor);
func_outputs.push_back(host_tensor);
}
bool ov::runtime::interpreter::INTExecutable::call(std::vector<ov::Tensor>& outputs,
const std::vector<ov::Tensor>& inputs) {
// map function params -> HostTensor
std::unordered_map<std::shared_ptr<ov::descriptor::Tensor>, shared_ptr<HostTensor>> tensor_map;
std::unordered_map<std::shared_ptr<ov::descriptor::Tensor>, ov::Tensor> tensor_map;
size_t input_count = 0;
for (const auto& param : get_parameters()) {
for (size_t i = 0; i < param->get_output_size(); ++i) {
auto tensor = param->output(i).get_tensor_ptr();
tensor_map.insert({tensor, func_inputs[input_count++]});
tensor_map.insert({tensor, inputs[input_count++]});
}
}
@ -96,186 +129,138 @@ bool runtime::interpreter::INTExecutable::call(const vector<shared_ptr<runtime::
// for each ordered op in the graph
for (const auto& op : m_nodes) {
if (dynamic_pointer_cast<op::Parameter>(op) != nullptr) {
if (std::dynamic_pointer_cast<ov::op::v0::Parameter>(op)) {
continue;
}
// get op inputs from map
vector<shared_ptr<HostTensor>> op_inputs;
std::vector<ov::Tensor> op_inputs;
for (auto input : op->inputs()) {
auto tensor = input.get_tensor_ptr();
op_inputs.push_back(tensor_map.at(tensor));
}
TemporaryOverrideOutputs overrider(op, op_inputs);
OutputVector outputs;
OutputVector output_ports;
for (size_t i = 0; i < op->inputs().size(); ++i) {
outputs.push_back(op->get_input_source_output(i));
output_ports.push_back(op->get_input_source_output(i));
}
auto cloned_node = op->clone_with_new_inputs(outputs);
auto cloned_node = op->clone_with_new_inputs(output_ports);
// get op outputs from map or create
vector<shared_ptr<HostTensor>> op_outputs;
std::vector<ov::Tensor> op_outputs;
for (size_t i = 0; i < op->get_output_size(); ++i) {
auto tensor = op->output(i).get_tensor_ptr();
shared_ptr<HostTensor> host_tensor;
ov::Tensor host_tensor;
auto it = tensor_map.find(tensor);
if (op::is_output(op)) {
host_tensor = func_outputs[results_map[tensor]];
} else if (it == tensor_map.end()) {
// Use cloned_node to create HostTensor with static dimensions
host_tensor = make_shared<HostTensor>(cloned_node->output(i));
tensor_map.insert({tensor, host_tensor});
auto output = cloned_node->output(i);
if (op::util::is_output(op) || it == tensor_map.end() || !it->second) {
host_tensor = ov::Tensor(output.get_element_type(),
output.get_partial_shape().is_dynamic()
? ov::Shape{0, std::numeric_limits<size_t>::max()}
: output.get_shape());
} else {
host_tensor = it->second;
}
op_outputs.push_back(host_tensor);
}
if (m_performance_counters_enabled) {
m_timer_map[op].start();
}
if (auto var_extension = std::dynamic_pointer_cast<ov::op::util::VariableExtension>(cloned_node)) {
auto variable = var_extension->get_variable();
if (!variable_context.get_variable_value(variable)) {
auto h_tensor = std::make_shared<ngraph::HostTensor>(cloned_node->get_input_element_type(0),
cloned_node->get_input_shape(0));
h_tensor->write(h_tensor->get_data_ptr(), h_tensor->get_size_in_bytes());
variable_context.set_variable_value(variable, std::make_shared<VariableValue>(h_tensor));
auto h_tensor = ov::Tensor(cloned_node->get_input_element_type(0), cloned_node->get_input_shape(0));
// h_tensor->write(h_tensor->get_data_ptr(), h_tensor->get_size_in_bytes());
const auto tensor_input = make_tmp_host_tensor(h_tensor);
variable_context.set_variable_value(variable,
std::make_shared<ov::op::util::VariableValue>(tensor_input));
}
}
const auto tensor_inputs = ov::util::wrap_tensors(op_inputs);
auto tensor_outputs = ov::util::wrap_tensors(op_outputs);
// Call evaluate for cloned_node with static shapes
if (cloned_node->evaluate(tensor_outputs, tensor_inputs, eval_context)) {
ov::util::update_output_host_tensors(op_outputs, tensor_outputs);
} else {
if (!cloned_node->evaluate(op_outputs, op_inputs, eval_context)) {
evaluate_node(cloned_node, op_outputs, op_inputs);
}
if (m_performance_counters_enabled) {
m_timer_map[op].stop();
}
if (m_nan_check_enabled) {
perform_nan_check(op_outputs, op.get());
// Update tensors in tensor map
for (size_t i = 0; i < op->get_output_size(); ++i) {
auto tensor = op->output(i).get_tensor_ptr();
tensor_map.insert({tensor, op_outputs[i]});
if (op::util::is_output(op)) {
auto& output = outputs[results_map[tensor]];
if (!output || output.get_shape() != op_outputs[i].get_shape()) {
outputs[results_map[tensor]] = op_outputs[i];
} else {
op_outputs[i].copy_to(output);
}
}
}
}
return true;
}
vector<runtime::PerformanceCounter> runtime::interpreter::INTExecutable::get_performance_data() const {
vector<runtime::PerformanceCounter> rc;
for (const pair<shared_ptr<const Node>, stopwatch> p : m_timer_map) {
rc.emplace_back(p.first, p.second.get_total_microseconds(), p.second.get_call_count());
}
return rc;
}
void runtime::interpreter::INTExecutable::perform_nan_check(const vector<shared_ptr<HostTensor>>& tensors,
const Node* op) {
size_t arg_number = 1;
for (const shared_ptr<HostTensor>& tensor : tensors) {
const element::Type& type = tensor->get_element_type();
if (type == element::f32) {
const float* data = tensor->get_data_ptr<float>();
for (size_t i = 0; i < tensor->get_element_count(); i++) {
if (std::isnan(data[i])) {
if (op) {
throw runtime_error("nan found in op '" + op->get_name() + "' output");
} else {
throw runtime_error("nan found in function's input tensor number " + to_string(arg_number));
}
}
}
} else if (type == element::f64) {
const double* data = tensor->get_data_ptr<double>();
for (size_t i = 0; i < tensor->get_element_count(); i++) {
if (std::isnan(data[i])) {
if (op) {
throw runtime_error("nan found in op '" + op->get_name() + "' output");
} else {
throw runtime_error("nan found in function's input tensor number " + to_string(arg_number));
}
}
}
}
arg_number++;
}
}
shared_ptr<ngraph::op::Parameter> runtime::interpreter::INTExecutable::get_parameter(size_t index) const {
std::shared_ptr<ov::op::v0::Parameter> ov::runtime::interpreter::INTExecutable::get_parameter(size_t index) const {
const ParameterVector& parameters = get_parameters();
NGRAPH_CHECK(index < parameters.size(), "create_tensor for input out of bounds");
return parameters[index];
}
shared_ptr<ngraph::op::Result> runtime::interpreter::INTExecutable::get_result(size_t index) const {
std::shared_ptr<ov::op::v0::Result> ov::runtime::interpreter::INTExecutable::get_result(size_t index) const {
const ResultVector& results = get_results();
NGRAPH_CHECK(index < results.size(), "create_tensor for input out of bounds");
return results[index];
}
shared_ptr<runtime::Tensor> runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index) {
shared_ptr<op::Parameter> parameter = get_parameter(input_index);
return make_shared<runtime::HostTensor>(parameter->get_element_type(), parameter->get_shape());
ov::Tensor ov::runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index) {
std::shared_ptr<op::v0::Parameter> parameter = get_parameter(input_index);
return ov::Tensor(parameter->get_element_type(), parameter->get_shape());
}
shared_ptr<runtime::Tensor> runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index) {
shared_ptr<op::Result> result = get_result(output_index);
return make_shared<runtime::HostTensor>(result->get_element_type(), result->get_shape());
ov::Tensor ov::runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index) {
std::shared_ptr<op::v0::Result> result = get_result(output_index);
return ov::Tensor(result->get_element_type(), result->get_shape());
}
vector<shared_ptr<runtime::Tensor>> runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index,
size_t pipeline_depth) {
vector<shared_ptr<runtime::HostTensor>> tensors;
shared_ptr<op::Parameter> parameter = get_parameter(input_index);
std::vector<ov::Tensor> ov::runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index,
size_t pipeline_depth) {
std::vector<ov::Tensor> tensors;
std::shared_ptr<op::v0::Parameter> parameter = get_parameter(input_index);
for (size_t i = 0; i < pipeline_depth; i++) {
shared_ptr<runtime::HostTensor> tensor;
auto t = make_shared<runtime::HostTensor>(parameter->get_element_type(), parameter->get_shape());
tensor = static_pointer_cast<runtime::HostTensor>(t);
tensors.push_back(tensor);
ov::Tensor tensor;
auto t = ov::Tensor(parameter->get_element_type(), parameter->get_shape());
tensors.push_back(t);
}
vector<shared_ptr<runtime::Tensor>> result_tensors;
for (const shared_ptr<runtime::HostTensor>& tensor : tensors) {
result_tensors.push_back(tensor);
}
return result_tensors;
return tensors;
}
vector<shared_ptr<runtime::Tensor>> runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index,
size_t pipeline_depth) {
vector<shared_ptr<runtime::HostTensor>> tensors;
shared_ptr<op::Result> result = get_result(output_index);
std::vector<ov::Tensor> ov::runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index,
size_t pipeline_depth) {
std::vector<ov::Tensor> tensors;
std::shared_ptr<op::v0::Result> result = get_result(output_index);
for (size_t i = 0; i < pipeline_depth; i++) {
shared_ptr<runtime::HostTensor> tensor;
auto t = make_shared<runtime::HostTensor>(result->get_element_type(), result->get_shape());
tensor = static_pointer_cast<runtime::HostTensor>(t);
tensors.push_back(tensor);
ov::Tensor tensor;
auto t = ov::Tensor(result->get_element_type(), result->get_shape());
tensors.push_back(t);
}
vector<shared_ptr<runtime::Tensor>> result_tensors;
for (const shared_ptr<runtime::HostTensor>& tensor : tensors) {
result_tensors.push_back(tensor);
}
return result_tensors;
return tensors;
}
bool runtime::interpreter::INTExecutable::evaluate_node(const std::shared_ptr<Node>& node,
const HostTensorVector& outputs,
const HostTensorVector& inputs) const {
auto& map = runtime::interpreter::get_evaluators_map();
bool ov::runtime::interpreter::INTExecutable::evaluate_node(const std::shared_ptr<Node>& node,
ov::TensorVector& outputs,
const ov::TensorVector& inputs) const {
auto& map = ngraph::runtime::interpreter::get_evaluators_map();
auto it = map.find(node->get_type_info());
bool res = false;
const auto tensor_inputs = create_tmp_tensors(inputs);
auto tensor_outputs = create_tmp_tensors(outputs);
if (it != map.end()) {
res = it->second(node, outputs, inputs);
res = it->second(node, tensor_outputs, tensor_inputs);
if (!res) {
throw ngraph_error(std::string("Running evaluate method for OP ") + node->get_type_info().name +
std::string(" failed!"));
throw ngraph::ngraph_error(std::string("Running evaluate method for OP ") + node->get_type_info().name +
std::string(" failed!"));
}
update_output_tensors(outputs, tensor_outputs);
} else {
throw unsupported_op(std::string("Interpreter backend doesn't implement evaluate method for OP ") +
node->get_type_info().name);
throw ngraph::unsupported_op(std::string("Interpreter backend doesn't implement evaluate method for OP ") +
node->get_type_info().name);
}
return res;
}

61
src/plugins/template/backend/int_executable.hpp
View File

@ -7,68 +7,47 @@
#include <initializer_list>
#include <iostream>
#include <memory>
#include <ngraph/runtime/host_tensor.hpp>
#include <sstream>
#include <string>
#include <vector>
#include "backend.hpp"
#include "ngraph/ops.hpp"
#include "ngraph/runtime/aligned_buffer.hpp"
#include "ngraph/runtime/reference/hard_sigmoid.hpp"
#include "ngraph/runtime/reference/non_max_suppression.hpp"
#include "ngraph/runtime/reference/reorg_yolo.hpp"
#include "ngraph/runtime/reference/tensor_iterator.hpp"
#include "ngraph/runtime/tensor.hpp"
#include "openvino/core/model.hpp"
#include "openvino/op/non_max_suppression.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/op/result.hpp"
namespace ngraph {
namespace ov {
namespace runtime {
namespace interpreter {
class INTBackend;
class INTExecutable;
} // namespace interpreter
} // namespace runtime
} // namespace ngraph
class ngraph::runtime::interpreter::INTExecutable : public Executable {
class INTExecutable : public Executable {
friend class INTBackend;
public:
INTExecutable(const std::shared_ptr<Function>& function, bool enable_performance_collection = false);
INTExecutable(const std::shared_ptr<ov::Model>& model);
bool call(const std::vector<std::shared_ptr<Tensor>>& outputs,
const std::vector<std::shared_ptr<Tensor>>& inputs) override;
bool call(std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs) override;
void set_nan_check(bool enable);
ov::Tensor create_input_tensor(size_t input_index) override;
std::vector<PerformanceCounter> get_performance_data() const override;
ov::Tensor create_output_tensor(size_t output_index) override;
std::shared_ptr<runtime::Tensor> create_input_tensor(size_t input_index) override;
std::vector<ov::Tensor> create_input_tensor(size_t input_index, size_t pipeline_depth) override;
std::shared_ptr<runtime::Tensor> create_output_tensor(size_t output_index) override;
std::vector<std::shared_ptr<runtime::Tensor>> create_input_tensor(size_t input_index,
size_t pipeline_depth) override;
std::vector<std::shared_ptr<runtime::Tensor>> create_output_tensor(size_t output_index,
size_t pipeline_depth) override;
std::vector<ov::Tensor> create_output_tensor(size_t output_index, size_t pipeline_depth) override;
protected:
std::shared_ptr<ngraph::op::Parameter> get_parameter(size_t index) const;
std::shared_ptr<ngraph::op::Result> get_result(size_t index) const;
std::shared_ptr<ov::op::v0::Parameter> get_parameter(size_t index) const;
std::shared_ptr<ov::op::v0::Result> get_result(size_t index) const;
bool evaluate_node(const std::shared_ptr<Node>& node,
const HostTensorVector& outputs,
const HostTensorVector& inputs) const;
ov::TensorVector& outputs,
const ov::TensorVector& inputs) const;
bool m_is_compiled = false;
bool m_nan_check_enabled = false;
bool m_performance_counters_enabled = false;
std::shared_ptr<Function> m_function;
NGRAPH_SUPPRESS_DEPRECATED_START
std::unordered_map<std::shared_ptr<const Node>, stopwatch> m_timer_map;
NGRAPH_SUPPRESS_DEPRECATED_END
std::shared_ptr<ov::Model> m_model;
std::vector<std::shared_ptr<Node>> m_nodes;
static void perform_nan_check(const std::vector<std::shared_ptr<HostTensor>>&, const Node* op = nullptr);
struct InfoForNMS5 {
int64_t max_output_boxes_per_class;
float iou_threshold;
@ -84,6 +63,10 @@ protected:
ngraph::element::Type output_type;
};
InfoForNMS5 get_info_for_nms5_eval(const op::v5::NonMaxSuppression* nms5,
InfoForNMS5 get_info_for_nms5_eval(const ov::op::v5::NonMaxSuppression* nms5,
const std::vector<std::shared_ptr<HostTensor>>& inputs);
};
} // namespace interpreter
} // namespace runtime
} // namespace ov

128
src/plugins/template/backend/pass/dyn_elimination.cpp
View File

@ -1,128 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "dyn_elimination.hpp"
#include <numeric>
#include "ngraph/builder/reshape.hpp"
#include "ngraph/op/broadcast.hpp"
#include "ngraph/op/range.hpp"
#include "ngraph/op/transpose.hpp"
#include "ngraph/pattern/matcher.hpp"
#include "ngraph/pattern/op/label.hpp"
#include "ngraph/runtime/reference/range.hpp"
#include "ngraph/slice_plan.hpp"
NGRAPH_SUPPRESS_DEPRECATED_START
using namespace std;
using namespace ngraph;
pass::DynElimination::DynElimination() : GraphRewrite() {
construct_range();
}
template <typename T>
std::shared_ptr<op::Constant> make_range_replacement(const element::Type& et,
const Shape& shape,
const std::shared_ptr<op::Constant>& start_arg,
const std::shared_ptr<op::Constant>& step_arg) {
std::vector<T> elements(shape_size(shape));
std::vector<T> start_vec = start_arg->get_vector<T>();
std::vector<T> step_vec = step_arg->get_vector<T>();
NGRAPH_CHECK(start_vec.size() == 1 && step_vec.size() == 1);
runtime::reference::range<T>(start_vec.data(), step_vec.data(), shape_size(shape), elements.data());
return make_shared<op::Constant>(et, shape, elements);
}
void pass::DynElimination::construct_range() {
auto start_arg_label = make_shared<pattern::op::Label>(element::f32, Shape{}, pattern::has_class<op::Constant>());
auto stop_arg_label = make_shared<pattern::op::Label>(element::f32, Shape{}, pattern::has_class<op::Constant>());
auto step_arg_label = make_shared<pattern::op::Label>(element::f32, Shape{}, pattern::has_class<op::Constant>());
auto range_pat = make_shared<op::Range>(start_arg_label, stop_arg_label, step_arg_label);
auto range_callback = [start_arg_label, stop_arg_label, step_arg_label](pattern::Matcher& m) {
auto pattern_map = m.get_pattern_map();
auto start_arg = static_pointer_cast<op::Constant>(pattern_map[start_arg_label]);
auto step_arg = static_pointer_cast<op::Constant>(pattern_map[step_arg_label]);
auto range_node = static_pointer_cast<op::Range>(m.get_match_root());
NGRAPH_CHECK(start_arg->get_output_partial_shape(0).rank().compatible(0) &&
step_arg->get_output_partial_shape(0).rank().compatible(0));
auto et = range_node->get_output_element_type(0);
auto shape = range_node->get_output_shape(0);
std::shared_ptr<op::Constant> replacement;
#if defined(__GNUC__) && !(__GNUC__ == 4 && __GNUC_MINOR__ == 8)
# pragma GCC diagnostic push
# pragma GCC diagnostic error "-Wswitch"
# pragma GCC diagnostic error "-Wswitch-enum"
#endif
switch (et) {
case element::Type_t::bf16:
replacement = make_range_replacement<bfloat16>(et, shape, start_arg, step_arg);
break;
case element::Type_t::f16:
replacement = make_range_replacement<float16>(et, shape, start_arg, step_arg);
break;
case element::Type_t::f32:
replacement = make_range_replacement<float>(et, shape, start_arg, step_arg);
break;
case element::Type_t::f64:
replacement = make_range_replacement<double>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i8:
replacement = make_range_replacement<int8_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i16:
replacement = make_range_replacement<int16_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i32:
replacement = make_range_replacement<int32_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i64:
replacement = make_range_replacement<int64_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::u8:
replacement = make_range_replacement<uint8_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::u16:
replacement = make_range_replacement<uint16_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::u32:
replacement = make_range_replacement<uint32_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::u64:
replacement = make_range_replacement<uint64_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i4:
case element::Type_t::u4:
case element::Type_t::u1:
case element::Type_t::undefined:
case element::Type_t::dynamic:
case element::Type_t::boolean:
NGRAPH_CHECK(false, "Internal nGraph error: unsupported element type: ", et);
break;
}
#if defined(__GNUC__) && !(__GNUC__ == 4 && __GNUC_MINOR__ == 8)
# pragma GCC diagnostic pop
#endif
replace_node(range_node, replacement);
return true;
};
auto range_matcher = make_shared<pattern::Matcher>(range_pat, "DynElimination.Range");
NGRAPH_SUPPRESS_DEPRECATED_START
add_matcher(range_matcher, range_callback, all_pass_property_off);
NGRAPH_SUPPRESS_DEPRECATED_END
}

47
src/plugins/template/backend/pass/dyn_elimination.hpp
View File

@ -1,47 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "ngraph/pass/graph_rewrite.hpp"
#include "ngraph/util.hpp"
namespace ngraph {
namespace pass {
/// \brief The DynElimination pass finds dynamic operations in a graph whose
/// shape relevant inputs have already been resolved to static values, and
/// replaces those dynamic operations with the equivalent operations using
/// static inputs and attributes.
/// \details This pass should be executed after the ConstantFolding pass.
///
/// The ConstantFolding and DynElimination passes are used together to transform
/// dynamic operations in a computation graph to static operations when the
/// graph is executed with input data.
///
/// In the example shown below, the original graph is constructed with dynamic
/// broadcast operation. When the graph is executed with input data, the input
/// shapes become available, by applying the ConstantFolding and DynElimination
/// pass, the graph is updated with dynamic broadcast being replaced by a static
/// broadcast operation.
/// <table>
/// <tr>
/// <th>Original</th>
/// <th>After %ConstantFolding</th>
/// <th>After %DynElimination</th>
/// </tr>
/// <tr>
/// <td> \image html dyn_broadcast_pre_constfld.svg </td>
/// <td> \image html dyn_broadcast_post_constfld.svg </td>
/// <td> \image html dyn_broadcast_post_dyneliminate.svg </td>
/// </tr>
/// </table>
class DynElimination : public GraphRewrite {
public:
DynElimination();
private:
void construct_range();
};
} // namespace pass
} // namespace ngraph

98
src/plugins/template/backend/pass/shape_relevance.cpp
View File

@ -1,98 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "pass/shape_relevance.hpp"
#include "ngraph/graph_util.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/op/util/op_types.hpp"
using namespace ngraph;
//
// This pass refreshes the "is_relevant_to_shape" flag on each parameter. A parameter will be
// flagged as relevant to shapes if there is any path from that parameter to a shape-relevant
// input that does _not_ pass through a value-irrelevant input. For example:
//
// N0[Parameter] N1[Parameter]
// | |
// | |
// | |
// N2[v1::Reshape]
//
// N1 (but not N0) will be flagged as shape-relevant, because N1 feeds into the "shape" input
// of N2.
//
// N0[Parameter] N1[Parameter]
// | |
// | N2[ShapeOf]
// | |
// N3[v1::Reshape]
//
// Neither N0 nor N1 will be flagged as shape-relevant. (N1 does feed into the "shape" input of N3,
// but only via the value-irrelevant input of ShapeOf.)
//
bool pass::ShapeRelevance::run_on_model(const std::shared_ptr<Function>& f) {
// TODO(amprocte): We are probably reinventing the wheel with the graph traversal here; the
// reason is that we need to cut the traversal short in cases where input values are
// irrelevant. See if there is a way to reduce this duplication.
// Set of nodes that must be evaluated to determine the value of shape-relevant inputs.
std::set<Node*> shape_determinants;
// Step 1: Find root nodes (these are nodes with an output connected to a shape-relevant
// input).
for (auto& n : f->get_ops()) {
for (auto& output : n->outputs()) {
for (auto& input : output.get_target_inputs()) {
if (input.get_is_relevant_to_shapes()) {
shape_determinants.insert(n.get());
break;
}
}
}
}
// Step 2: Find all shape determinants. This is the transitive closure of R, where n1 R n2
// iff there is a data flow edge from n2 to n1 and that data flow edge is not
// value-irrelevant.
bool changes_made = false;
{
std::list<Node*> to_visit{shape_determinants.begin(), shape_determinants.end()};
std::set<Node*> already_visited;
while (!to_visit.empty()) {
auto node = to_visit.front();
to_visit.pop_front();
if (already_visited.count(node) > 0) {
continue;
}
shape_determinants.insert(node);
already_visited.insert(node);
if (op::is_parameter(node)) {
auto node_as_param = static_cast<op::Parameter*>(node);
if (!node_as_param->is_relevant_to_shapes()) {
node_as_param->set_is_relevant_to_shapes(true);
changes_made = true;
}
}
for (size_t i = 0; i < node->get_input_size(); i++) {
if (!node->input(i).get_is_relevant_to_values()) {
continue;
}
auto source_node = node->get_input_node_ptr(i);
if (already_visited.count(source_node) == 0) {
to_visit.push_front(source_node);
}
}
}
}
return changes_made;
}

17
src/plugins/template/backend/pass/shape_relevance.hpp
View File

@ -1,17 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "ngraph/pass/pass.hpp"
namespace ngraph {
namespace pass {
class ShapeRelevance : public FunctionPass {
public:
ShapeRelevance() : FunctionPass() {}
bool run_on_model(const std::shared_ptr<ngraph::Function>& m) override;
};
} // namespace pass
} // namespace ngraph

37
src/plugins/template/backend/performance_counter.hpp
View File

@ -1,37 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <cstddef>
#include <string>
#include "ngraph/node.hpp"
namespace ngraph {
namespace runtime {
class PerformanceCounter {
public:
PerformanceCounter(const std::shared_ptr<const Node>& n, size_t us, size_t calls)
: m_node(n),
m_total_microseconds(us),
m_call_count(calls) {}
std::shared_ptr<const Node> get_node() const {
return m_node;
}
size_t total_microseconds() const {
return m_total_microseconds;
}
size_t microseconds() const {
return m_call_count == 0 ? 0 : m_total_microseconds / m_call_count;
}
size_t call_count() const {
return m_call_count;
}
std::shared_ptr<const Node> m_node;
size_t m_total_microseconds;
size_t m_call_count;
};
} // namespace runtime
} // namespace ngraph

2
src/plugins/template/src/plugin.cpp
View File

@ -30,7 +30,7 @@ ov::template_plugin::Plugin::Plugin() {
set_device_name("TEMPLATE");
// create ngraph backend which performs inference using ngraph reference implementations
_backend = ngraph::runtime::Backend::create();
_backend = ov::runtime::Backend::create();
// create default stream executor with a given name
_waitExecutor = get_executor_manager()->get_idle_cpu_streams_executor({wait_executor_name});

2
src/plugins/template/src/plugin.hpp
View File

@ -50,7 +50,7 @@ private:
friend class CompiledModel;
friend class InferRequest;
std::shared_ptr<ngraph::runtime::Backend> _backend;
std::shared_ptr<ov::runtime::Backend> _backend;
Configuration _cfg;
std::shared_ptr<ov::threading::ITaskExecutor> _waitExecutor;
};

23
src/plugins/template/src/sync_infer_request.cpp
View File

@ -131,23 +131,7 @@ void ov::template_plugin::InferRequest::infer_preprocess() {
get_template_model()->get_template_plugin()->_backend->create_tensor(tensor.get_element_type(),
tensor.get_shape());
auto* src_data = static_cast<uint8_t*>(tensor.data());
auto dst_tensor = std::dynamic_pointer_cast<ngraph::runtime::HostTensor>(m_backend_input_tensors[i]);
OPENVINO_ASSERT(dst_tensor, "Template plugin error: Can't cast created tensor to HostTensor");
auto* dst_data = dst_tensor->get_data_ptr<uint8_t>();
std::vector<size_t> indexes(shape.size());
for (size_t dst_idx = 0; dst_idx < ov::shape_size(shape); dst_idx++) {
size_t val = dst_idx;
size_t src_idx = 0;
for (size_t j1 = 0; j1 < indexes.size(); j1++) {
size_t j = indexes.size() - j1 - 1;
indexes[j] = val % shape[j];
val /= shape[j];
src_idx += indexes[j] * tensor.get_strides()[j];
}
memcpy(dst_data + dst_idx * tensor.get_element_type().size(),
src_data + src_idx,
tensor.get_element_type().size());
}
tensor.copy_to(m_backend_input_tensors[i]);
}
}
// Tensors can be dynamic, so in this case we need to allocate tensors with right shape
@ -196,9 +180,8 @@ void ov::template_plugin::InferRequest::infer_postprocess() {
auto host_tensor = m_backend_output_tensors[i];
ov::Output<const ov::Node> output{result->output(0).get_node(), result->output(0).get_index()};
allocate_tensor(output, [host_tensor](ov::Tensor& tensor) {
allocate_tensor_impl(tensor, host_tensor->get_element_type(), host_tensor->get_shape());
// tensor.set_shape(host_tensor->get_shape());
host_tensor->read(static_cast<char*>(tensor.data()), host_tensor->get_size_in_bytes());
allocate_tensor_impl(tensor, host_tensor.get_element_type(), host_tensor.get_shape());
host_tensor.copy_to(tensor);
});
}
}

6
src/plugins/template/src/sync_infer_request.hpp
View File

@ -49,9 +49,9 @@ private:
// for performance counters
std::array<std::chrono::duration<float, std::micro>, numOfStages> m_durations;
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> m_backend_input_tensors;
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> m_backend_output_tensors;
std::shared_ptr<ngraph::runtime::Executable> m_executable;
std::vector<ov::Tensor> m_backend_input_tensors;
std::vector<ov::Tensor> m_backend_output_tensors;
std::shared_ptr<ov::runtime::Executable> m_executable;
};
// ! [infer_request:header]

40
src/tests/ngraph_helpers/ngraph_functions/src/utils/ngraph_helpers.cpp
View File

@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
//
#include <cstring>
#include <vector>
#include <memory>
#include <queue>
@ -16,6 +17,7 @@
#include <ngraph_functions/utils/ngraph_helpers.hpp>
#include <ngraph/opsets/opset.hpp>
#include <backend.hpp>
#include "openvino/runtime/tensor.hpp"
namespace ngraph {
namespace helpers {
@ -84,7 +86,7 @@ std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>
interpreterFunction(const std::shared_ptr<Function> &function,
const std::vector<std::vector<std::uint8_t>> &inputs,
const std::vector<ngraph::element::Type> &inputTypes) {
auto backend = runtime::Backend::create();
auto backend = ov::runtime::Backend::create();
const auto &parameters = function->get_parameters();
const auto &parametersNumber = parameters.size();
@ -98,7 +100,7 @@ std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>
inputTypes.size(), " types");
}
auto inputTensors = std::vector<std::shared_ptr<runtime::Tensor>>{};
ov::TensorVector inputTensors(parametersNumber);
for (size_t i = 0; i < parametersNumber; ++i) {
const auto &parameter = parameters[i];
const auto &parameterIndex = function->get_parameter_index(parameter);
@ -120,14 +122,14 @@ std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>
" has ", inputSize, " bytes");
auto tensor = backend->create_tensor(parameterType, parameterShape);
tensor->write(input.data(), parameterSize);
inputTensors.push_back(tensor);
std::memcpy(tensor.data(), input.data(), parameterSize);
inputTensors[i] = tensor;
}
auto outputTensors = std::vector<std::shared_ptr<runtime::Tensor>>{};
const auto &results = function->get_results();
ov::TensorVector outputTensors(results.size());
for (size_t i = 0; i < results.size(); ++i) {
outputTensors.push_back(std::make_shared<HostTensor>());
outputTensors[i] = ov::Tensor(results[i]->get_element_type(), {0});
}
auto handle = backend->compile(function);
@ -137,16 +139,16 @@ std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>
auto& output = outputs[resultIndex];
output.first = results[resultIndex]->get_element_type();
const auto& outputTensor = outputTensors[resultIndex];
output.second.resize((shape_size(outputTensor->get_shape()) * outputTensor->get_element_type().bitwidth() + 7) >> 3);
outputTensors[resultIndex]->read(output.second.data(), output.second.size());
output.second.resize((shape_size(outputTensor.get_shape()) * outputTensor.get_element_type().bitwidth() + 7) >> 3);
std::memcpy(output.second.data(), outputTensors[resultIndex].data(), output.second.size());
}
return outputs;
}
std::vector<ov::Tensor> interpretFunction(const std::shared_ptr<Function> &function,
const std::map<std::shared_ptr<ov::Node>, ov::Tensor>& inputs) {
auto backend = runtime::Backend::create();
const std::map<std::shared_ptr<ov::Node>, ov::Tensor>& inputs) {
auto backend = ov::runtime::Backend::create();
const auto &funcInputs = function->inputs();
const auto &funcInputsNumber = funcInputs.size();
@ -155,7 +157,7 @@ std::vector<ov::Tensor> interpretFunction(const std::shared_ptr<Function> &funct
"Got function (", function->get_friendly_name(), ") with ", funcInputsNumber, " parameters, but ",
inputsNumber, " input blobs");
auto inputTensors = std::vector<std::shared_ptr<runtime::Tensor>>{};
ov::TensorVector inputTensors(funcInputsNumber);
for (size_t i = 0; i < funcInputsNumber; ++i) {
const auto &input = funcInputs[i];
const auto &inputShape = input.get_shape();
@ -178,26 +180,20 @@ std::vector<ov::Tensor> interpretFunction(const std::shared_ptr<Function> &funct
" has ", inputTensorSize, " bytes");
auto tensor = backend->create_tensor(inputType, inputShape);
tensor->write(inputTensor.data(), inputSize);
inputTensors.push_back(tensor);
inputTensor.copy_to(tensor);
inputTensors[i] = tensor;
}
std::vector<std::shared_ptr<runtime::Tensor>> outputTensors;
const auto &results = function->get_results();
ov::TensorVector outputTensors(results.size());
for (size_t i = 0; i < results.size(); ++i) {
outputTensors.push_back(std::make_shared<HostTensor>());
outputTensors[i] = ov::Tensor(results[i]->get_element_type(), {0});
}
auto handle = backend->compile(function);
handle->call_with_validate(outputTensors, inputTensors);
std::vector<ov::Tensor> outputs;
for (const auto& outTensor : outputTensors) {
ov::Tensor tmpBuffer(outTensor->get_element_type(), outTensor->get_shape());
outTensor->read(tmpBuffer.data(), tmpBuffer.get_byte_size());
outputs.push_back(tmpBuffer);
}
return outputs;
return outputTensors;
}
std::shared_ptr<Function> foldFunction(const std::shared_ptr<Function> &function,