IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Moved template backend to new API (#15878)

Browse Source 
* Moved template backend to new API

* Fixed compilation

* Fixed some comments

* Fixed ov_core_unit_tests

* Fixed some tests

* Fixed ONNX Frontend tests

* Fixed transformation tests

* Fixed dynamic tests

* Fixed sporadic in CPU tests

* Added WA for plugin

* Fixed copy_to for scalar tensors
This commit is contained in:
Ilya Churaev 2023-03-01 07:12:33 +04:00 committed by GitHub
parent 87e714eb5c
commit e534efd4a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 331 additions and 1071 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/core/src/node.cpp
View File

@ -730,7 +730,8 @@ bool ov::Node::evaluate(ov::TensorVector& output_values, const ov::TensorVector&
OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_SUPPRESS_DEPRECATED_START
bool sts = evaluate(output, input); bool sts = evaluate(output, input);
OPENVINO_SUPPRESS_DEPRECATED_END OPENVINO_SUPPRESS_DEPRECATED_END
update_output_tensors(output_values, output); if (sts)
update_output_tensors(output_values, output);
return sts; return sts;
} }
@ -743,7 +744,8 @@ bool ov::Node::evaluate(ov::TensorVector& output_values,
OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_SUPPRESS_DEPRECATED_START
bool sts = evaluate(output, input, evaluationContext); bool sts = evaluate(output, input, evaluationContext);
OPENVINO_SUPPRESS_DEPRECATED_END OPENVINO_SUPPRESS_DEPRECATED_END
update_output_tensors(output_values, output); if (sts)
update_output_tensors(output_values, output);
// Call evaluate for ov::Tensor if op doesn't have evaluate with EvaluationContext // Call evaluate for ov::Tensor if op doesn't have evaluate with EvaluationContext
return sts ? sts : evaluate(output_values, input_values); return sts ? sts : evaluate(output_values, input_values);
} }

12
src/core/src/runtime/ov_tensor.cpp
View File

@ -128,6 +128,13 @@ Shape Tensor::get_shape() const {
} }
void Tensor::copy_to(ov::Tensor& dst) const { void Tensor::copy_to(ov::Tensor& dst) const {
const auto& is_scalar = [](const ov::Shape& shape) {
return shape.empty() || (shape.size() == 1 && shape[0] == 1);
};
const auto shapes_equal = [is_scalar](const ov::Shape& src, const ov::Shape& dst) {
// WA for scalar tensors to copy {1} to {} or otherwise
return src == dst || (is_scalar(src) && is_scalar(dst));
};
OV_TENSOR_STATEMENT({ OV_TENSOR_STATEMENT({
OPENVINO_ASSERT(dst, "Destination tensor was not initialized."); OPENVINO_ASSERT(dst, "Destination tensor was not initialized.");
OPENVINO_ASSERT(!is<ov::RemoteTensor>(), "Default copy to doesn't support copy from remote tensor."); OPENVINO_ASSERT(!is<ov::RemoteTensor>(), "Default copy to doesn't support copy from remote tensor.");
@ -140,7 +147,7 @@ void Tensor::copy_to(ov::Tensor& dst) const {
")"); ")");
if (dst.get_shape() == ov::Shape{0}) if (dst.get_shape() == ov::Shape{0})
dst.set_shape(get_shape()); dst.set_shape(get_shape());
OPENVINO_ASSERT(dst.get_shape() == get_shape(), OPENVINO_ASSERT(shapes_equal(get_shape(), dst.get_shape()),
"Tensor shapes are not equal. (src: ", "Tensor shapes are not equal. (src: ",
get_shape(), get_shape(),
" != dst: ", " != dst: ",
@ -154,7 +161,8 @@ void Tensor::copy_to(ov::Tensor& dst) const {
ov::Shape cur_pos{0}; ov::Shape cur_pos{0};
ov::Shape max_pos{1}; ov::Shape max_pos{1};
if (get_element_type().bitwidth() < 8 || (get_strides() == dst.get_strides() && is_continuous())) { if (get_element_type().bitwidth() < 8 || (get_strides() == dst.get_strides() && is_continuous()) ||
(is_scalar(get_shape()) && is_scalar(dst.get_shape()))) {
// OpenVINO doesn't support strides for LP types // OpenVINO doesn't support strides for LP types
// or both tensors have default strides // or both tensors have default strides
// Strides and positions already initialized // Strides and positions already initialized

20
src/core/tests/ov_tensor_test.cpp
View File

@ -323,8 +323,10 @@ std::vector<T> fill_data(const ov::Tensor& tensor) {
const T* data = tensor.data<T>(); const T* data = tensor.data<T>();
auto strides = tensor.get_strides(); auto strides = tensor.get_strides();
for (auto&& c : ngraph::CoordinateTransformBasic{tensor.get_shape()}) { for (auto&& c : ngraph::CoordinateTransformBasic{tensor.get_shape()}) {
actual.emplace_back( size_t offset = 0;
*(data + (c[2] * strides[2] + c[1] * strides[1] + c[0] * strides[0]) / tensor.get_element_type().size())); for (size_t i = 0; i < strides.size(); i++)
offset += c[i] * strides[i];
actual.emplace_back(*(data + offset / tensor.get_element_type().size()));
} }
return actual; return actual;
}; };
@ -394,7 +396,7 @@ void init_tensor(const ov::Tensor& tensor, bool input) {
void compare_tensors(const ov::Tensor& src, const ov::Tensor& dst) { void compare_tensors(const ov::Tensor& src, const ov::Tensor& dst) {
ASSERT_EQ(src.get_byte_size(), dst.get_byte_size()); ASSERT_EQ(src.get_byte_size(), dst.get_byte_size());
ASSERT_EQ(src.get_shape(), dst.get_shape()); ASSERT_EQ(src.get_size(), dst.get_size());
ASSERT_EQ(src.get_element_type(), dst.get_element_type()); ASSERT_EQ(src.get_element_type(), dst.get_element_type());
switch (src.get_element_type()) { switch (src.get_element_type()) {
case ov::element::bf16: case ov::element::bf16:
@ -501,6 +503,18 @@ INSTANTIATE_TEST_SUITE_P(copy_tests,
TestParams { TestParams {
ov::Shape{3, 2, 2}, ov::Strides{64, 16, 8}, ov::Shape{3, 2, 2}, ov::Strides{64, 16, 8},
ov::Shape{3, 2, 2}, ov::Strides{128, 24, 8} ov::Shape{3, 2, 2}, ov::Strides{128, 24, 8}
},
TestParams {
ov::Shape{}, {},
{}, {}
},
TestParams {
ov::Shape{1}, {},
{}, {}
},
TestParams {
ov::Shape{}, {},
{1}, {}
} }
))); )));
// clang-format on // clang-format on

11
src/plugins/template/backend/CMakeLists.txt
View File

@ -5,16 +5,11 @@
set (SRC set (SRC
backend.cpp backend.cpp
backend.hpp backend.hpp
cache.cpp
cache.hpp
executable.cpp executable.cpp
executable.hpp executable.hpp
performance_counter.hpp int_backend.cpp
pass/dyn_elimination.cpp int_executable.cpp
pass/dyn_elimination.hpp evaluates_map.cpp
pass/shape_relevance.cpp
pass/shape_relevance.hpp
int_backend.cpp int_executable.cpp evaluates_map.cpp
) )
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")

31
src/plugins/template/backend/backend.cpp
View File

@ -9,36 +9,13 @@
#include "int_backend.hpp" #include "int_backend.hpp"
#include "ngraph/file_util.hpp" #include "ngraph/file_util.hpp"
#include "ngraph/util.hpp" #include "ngraph/util.hpp"
#include "openvino/core/except.hpp"
#include "openvino/util/file_util.hpp" #include "openvino/util/file_util.hpp"
using namespace std; ov::runtime::Backend::~Backend() = default;
using namespace ngraph;
runtime::Backend::~Backend() {} std::shared_ptr<ov::runtime::Backend> ov::runtime::Backend::create() {
auto inner_backend = std::make_shared<interpreter::INTBackend>();
std::shared_ptr<runtime::Backend> runtime::Backend::create() {
auto inner_backend = make_shared<interpreter::INTBackend>();
return inner_backend; return inner_backend;
} }
std::shared_ptr<ngraph::runtime::Tensor> runtime::Backend::create_dynamic_tensor(
const ngraph::element::Type& /* element_type */,
const PartialShape& /* shape */) {
throw std::invalid_argument("This backend does not support dynamic tensors");
}
bool runtime::Backend::is_supported(const Node& /* node */) const {
// The default behavior is that a backend does not support any ops. If this is not the case
// then override this method and enhance.
return false;
}
std::shared_ptr<runtime::Executable> runtime::Backend::load(istream& /* input_stream */) {
throw runtime_error("load operation unimplemented.");
}
bool runtime::Backend::set_config(const map<string, string>& /* config */, string& error) {
error = "set_config not supported";
return false;
}

72
src/plugins/template/backend/backend.hpp
View File

@ -8,23 +8,16 @@
#include <mutex> #include <mutex>
#include "executable.hpp" #include "executable.hpp"
#include "ngraph/function.hpp" #include "openvino/core/model.hpp"
#include "ngraph/shape.hpp" #include "openvino/core/type/element_type.hpp"
#include "ngraph/type/element_type.hpp"
#include "ngraph/util.hpp"
#include "performance_counter.hpp"
namespace ngraph { namespace ov {
namespace runtime { namespace runtime {
class Tensor;
class Backend;
} // namespace runtime
} // namespace ngraph
/// \brief Interface to a generic backend. /// \brief Interface to a generic backend.
/// ///
/// Backends are responsible for function execution and value allocation. /// Backends are responsible for function execution and value allocation.
class ngraph::runtime::Backend { class Backend {
public: public:
virtual ~Backend(); virtual ~Backend();
/// \brief Create a new Backend object /// \brief Create a new Backend object
@ -44,14 +37,13 @@ public:
/// passed as an output to a function the tensor will have a type and shape after executing /// passed as an output to a function the tensor will have a type and shape after executing
/// a call. /// a call.
/// \returns shared_ptr to a new backend-specific tensor /// \returns shared_ptr to a new backend-specific tensor
virtual std::shared_ptr<ngraph::runtime::Tensor> create_tensor() = 0; virtual ov::Tensor create_tensor() = 0;
/// \brief Create a tensor specific to this backend /// \brief Create a tensor specific to this backend
/// \param element_type The type of the tensor element /// \param element_type The type of the tensor element
/// \param shape The shape of the tensor /// \param shape The shape of the tensor
/// \returns shared_ptr to a new backend-specific tensor /// \returns shared_ptr to a new backend-specific tensor
virtual std::shared_ptr<ngraph::runtime::Tensor> create_tensor(const ngraph::element::Type& element_type, virtual ov::Tensor create_tensor(const ov::element::Type& element_type, const Shape& shape) = 0;
const Shape& shape) = 0;
/// \brief Create a tensor specific to this backend /// \brief Create a tensor specific to this backend
/// \param element_type The type of the tensor element /// \param element_type The type of the tensor element
@ -60,59 +52,23 @@ public:
/// must be sufficient to contain the tensor. The lifetime of the buffer is the /// must be sufficient to contain the tensor. The lifetime of the buffer is the
/// responsibility of the caller. /// responsibility of the caller.
/// \returns shared_ptr to a new backend-specific tensor /// \returns shared_ptr to a new backend-specific tensor
virtual std::shared_ptr<ngraph::runtime::Tensor> create_tensor(const ngraph::element::Type& element_type, virtual ov::Tensor create_tensor(const ov::element::Type& element_type,
const Shape& shape, const Shape& shape,
void* memory_pointer) = 0; void* memory_pointer) = 0;
/// \brief Create a tensor of C type T specific to this backend /// \brief Create a tensor of C type T specific to this backend
/// \param shape The shape of the tensor /// \param shape The shape of the tensor
/// \returns shared_ptr to a new backend specific tensor /// \returns shared_ptr to a new backend specific tensor
template <typename T> template <typename T>
std::shared_ptr<ngraph::runtime::Tensor> create_tensor(const Shape& shape) { ov::Tensor create_tensor(const Shape& shape) {
return create_tensor(element::from<T>(), shape); return create_tensor(element::from<T>(), shape);
} }
/// \brief Create a dynamic tensor specific to this backend, if the backend supports dynamic
/// tensors.
/// \param element_type The type of the tensor element
/// \param shape The shape of the tensor
/// \returns shared_ptr to a new backend-specific tensor
/// \throws std::invalid_argument if the backend does not support dynamic tensors
virtual std::shared_ptr<ngraph::runtime::Tensor> create_dynamic_tensor(const ngraph::element::Type& element_type,
const PartialShape& shape);
/// \returns `true` if this backend supports dynamic tensors, else `false`.
virtual bool supports_dynamic_tensors() {
return false;
}
/// \brief Compiles a Function. /// \brief Compiles a Function.
/// \param func The function to compile /// \param func The function to compile
/// \returns compiled function or nullptr on failure /// \returns compiled function or nullptr on failure
virtual std::shared_ptr<Executable> compile(std::shared_ptr<Function> func, virtual std::shared_ptr<Executable> compile(std::shared_ptr<ov::Model> model) = 0;
bool enable_performance_data = false) = 0;
/// \brief Loads a previously saved Executable object from a stream.
/// \param input_stream the opened input stream containing the saved Executable
/// \returns A compiled function or throws an exception on error
virtual std::shared_ptr<Executable> load(std::istream& input_stream);
/// \brief Test if a backend is capable of supporting an op
/// \param node is the op to test.
/// \returns true if the op is supported, false otherwise.
virtual bool is_supported(const Node& node) const;
/// \brief Allows sending backend specific configuration. The map contains key, value pairs
/// specific to a particluar backend. The definition of these key, value pairs is
/// defined by each backend.
/// \param config The configuration map sent to the backend
/// \param error An error string describing any error encountered
/// \returns true if the configuration is supported, false otherwise. On false the error
/// parameter value is valid.
virtual bool set_config(const std::map<std::string, std::string>& config, std::string& error);
/// \brief Get the version of the backend
/// The default value of 0.0.0 is chosen to be a parsable version number
virtual std::string get_version() const {
return "0.0.0";
}
}; };
} // namespace runtime
} // namespace ov

95
src/plugins/template/backend/cache.cpp
View File

@ -1,95 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "cache.hpp"
#include "ngraph/env_util.hpp"
using namespace ngraph;
using namespace std;
// Constructor
runtime::LRUCache::LRUCache() {
m_cache_size = 1024;
m_map = {};
m_list = {};
}
// Destructor
runtime::LRUCache::~LRUCache() {
m_list.clear();
m_map.clear();
m_clone_function_map.clear();
}
void runtime::LRUCache::convert_shape_to_string(const vector<int>& shape, ostringstream& key) {
if (!shape.empty()) {
std::copy(shape.begin(), shape.end(), std::ostream_iterator<int>(key, ", "));
}
}
void runtime::LRUCache::add_entry(const vector<int>& shape,
shared_ptr<runtime::Executable> exec,
shared_ptr<Function> func) {
std::lock_guard<std::mutex> guard(m_mutex);
ostringstream key;
// check if the list is empty
if (m_list.size() == static_cast<size_t>(m_cache_size)) {
ostringstream key;
convert_shape_to_string(m_list.back(), key);
m_list.pop_back();
m_map.erase(key.str());
}
convert_shape_to_string(shape, key);
m_map.insert({key.str(), exec});
m_list.push_front(shape);
m_clone_function_map.insert({key.str(), func});
}
bool runtime::LRUCache::is_cached(const vector<int>& shape) {
for (auto itr = m_list.begin(); itr != m_list.end(); itr++) {
if (*itr == shape) {
return true;
}
}
return false;
}
shared_ptr<runtime::Executable> runtime::LRUCache::get_cached_entry(const vector<int>& shape) {
std::lock_guard<std::mutex> guard(m_mutex);
ostringstream key;
convert_shape_to_string(shape, key);
// find the entry and return the function
auto it = m_map.find(key.str());
if (it == m_map.end()) {
throw ngraph_error("Entry not found in cache");
} else {
// update list to push this reference to the front
for (auto itr = m_list.begin(); itr != m_list.end(); itr++) {
if (*itr == shape) {
m_list.remove(shape);
m_list.push_front(shape);
break;
}
}
return it->second;
}
}
// Need the clone function to get the output shape so that
// storage can be allocated for output
shared_ptr<Function> runtime::LRUCache::get_cloned_function(const vector<int>& shape) {
std::lock_guard<std::mutex> guard(m_mutex);
ostringstream key;
convert_shape_to_string(shape, key);
// find the entry and return the function
auto it = m_clone_function_map.find(key.str());
if (it == m_clone_function_map.end()) {
throw ngraph_error("Cloned function not found");
}
return it->second;
}

45
src/plugins/template/backend/cache.hpp
View File

@ -1,45 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <algorithm>
#include <iostream>
#include <iterator>
#include <list>
#include <mutex>
#include <sstream>
#include <string>
#include <unordered_map>
#include "executable.hpp"
#include "ngraph/function.hpp"
#include "ngraph/shape.hpp"
namespace ngraph {
namespace runtime {
class LRUCache : public std::enable_shared_from_this<LRUCache> {
public:
using GraphCache = std::unordered_map<std::string, std::shared_ptr<Executable>>;
using ClonedFunctionMap = std::unordered_map<std::string, std::shared_ptr<Function>>;
LRUCache();
virtual ~LRUCache();
void add_entry(const std::vector<int>& shape, std::shared_ptr<Executable> exec, std::shared_ptr<Function> func);
bool is_cached(const std::vector<int>& shape);
std::shared_ptr<Executable> get_cached_entry(const std::vector<int>& shape);
void convert_shape_to_string(const std::vector<int>& shape, std::ostringstream& key);
std::shared_ptr<Function> get_cloned_function(const std::vector<int>& shape);
private:
int m_cache_size;
GraphCache m_map;
ClonedFunctionMap m_clone_function_map;
std::list<std::vector<int>> m_list;
std::mutex m_mutex;
};
} // namespace runtime
} // namespace ngraph

44
src/plugins/template/backend/evaluates_map.cpp
View File

@ -95,6 +95,7 @@
#include "ngraph/runtime/reference/convert_color_nv12.hpp" #include "ngraph/runtime/reference/convert_color_nv12.hpp"
#include "ov_ops/augru_cell.hpp" #include "ov_ops/augru_cell.hpp"
#include "ov_ops/augru_sequence.hpp" #include "ov_ops/augru_sequence.hpp"
#include "tensor_conversion_util.hpp"
using namespace ngraph; using namespace ngraph;
using namespace std; using namespace std;
@ -3302,46 +3303,21 @@ runtime::reference::custom_evaluate_function evaluate = [](const std::shared_ptr
inputsNumber, inputsNumber,
" input blobs"); " input blobs");
auto inputTensors = std::vector<std::shared_ptr<runtime::Tensor>>{};
for (const auto& parameter : parameters) {
const auto& parameterIndex = function->get_parameter_index(parameter);
const auto& parameterShape = parameter->get_shape();
const auto& parameterType = parameter->get_element_type();
const auto& parameterSize = shape_size(parameterShape) * parameterType.size();
const auto& input = inputs[parameterIndex];
const auto& inputSize = input->get_size_in_bytes();
NGRAPH_CHECK(parameterSize == inputSize,
"Got parameter (",
parameter->get_friendly_name(),
") of size ",
parameterSize,
" bytes, but corresponding input with index ",
parameterIndex,
" has ",
inputSize,
" bytes");
auto tensor = std::make_shared<runtime::HostTensor>(parameterType, parameterShape);
tensor->write(input->get_data_ptr(), parameterSize);
inputTensors.push_back(tensor);
}
const auto& results = function->get_results(); const auto& results = function->get_results();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> outputTensors; outputs.reserve(results.size());
outputTensors.reserve(results.size());
for (size_t i = 0; i < results.size(); ++i) { for (size_t i = 0; i < results.size(); ++i) {
outputTensors.push_back(std::make_shared<HostTensor>()); outputs.push_back(std::make_shared<HostTensor>(results[i]->output(0)));
} }
auto backend = runtime::Backend::create();
auto backend = ov::runtime::Backend::create();
auto handle = backend->compile(function); auto handle = backend->compile(function);
OPENVINO_SUPPRESS_DEPRECATED_START
auto outputTensors = ov::util::wrap_tensors(outputs);
auto inputTensors = ov::util::wrap_tensors(inputs);
handle->call_with_validate(outputTensors, inputTensors); handle->call_with_validate(outputTensors, inputTensors);
outputs.reserve(outputTensors.size()); ov::util::update_output_host_tensors(outputs, outputTensors);
for (const auto& tensor : outputTensors) { OPENVINO_SUPPRESS_DEPRECATED_END
auto host_tensor = static_pointer_cast<runtime::HostTensor>(tensor);
outputs.push_back(host_tensor);
}
}; };
} // namespace ti_v0 } // namespace ti_v0

120
src/plugins/template/backend/executable.cpp
View File

@ -6,133 +6,81 @@
#include <sstream> #include <sstream>
#include "ngraph/file_util.hpp" #include "openvino/core/except.hpp"
#include "ngraph/runtime/tensor.hpp"
#include "ngraph/util.hpp"
using namespace std; ov::runtime::Executable::Executable() {}
using namespace ngraph;
runtime::Executable::Executable() {} ov::runtime::Executable::~Executable() {}
runtime::Executable::~Executable() {} bool ov::runtime::Executable::call_with_validate(std::vector<ov::Tensor>& outputs,
const std::vector<ov::Tensor>& inputs) {
bool runtime::Executable::call_with_validate(const vector<shared_ptr<runtime::Tensor>>& outputs,
const vector<shared_ptr<runtime::Tensor>>& inputs) {
validate(outputs, inputs); validate(outputs, inputs);
return call(outputs, inputs); return call(outputs, inputs);
} }
void runtime::Executable::validate(const vector<std::shared_ptr<runtime::Tensor>>& outputs, void ov::runtime::Executable::validate(const std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs) {
const vector<std::shared_ptr<runtime::Tensor>>& inputs) {
const ParameterVector& parameters = get_parameters(); const ParameterVector& parameters = get_parameters();
const ResultVector& results = get_results(); const ResultVector& results = get_results();
if (parameters.size() != inputs.size()) { if (parameters.size() != inputs.size()) {
stringstream ss; std::stringstream ss;
ss << "Call input count " << inputs.size() << " does not match Function's Parameter count " ss << "Call input count " << inputs.size() << " does not match Function's Parameter count "
<< parameters.size(); << parameters.size();
throw runtime_error(ss.str()); throw std::runtime_error(ss.str());
} }
if (results.size() != outputs.size()) { if (results.size() != outputs.size()) {
stringstream ss; std::stringstream ss;
ss << "Call output count " << outputs.size() << " does not match Function's Result count " << results.size(); ss << "Call output count " << outputs.size() << " does not match Function's Result count " << results.size();
throw runtime_error(ss.str()); throw std::runtime_error(ss.str());
} }
for (size_t i = 0; i < parameters.size(); i++) { for (size_t i = 0; i < parameters.size(); i++) {
if (parameters[i]->get_element_type().is_static() && if (parameters[i]->get_element_type().is_static() &&
parameters[i]->get_element_type() != inputs[i]->get_element_type()) { parameters[i]->get_element_type() != inputs[i].get_element_type()) {
stringstream ss; std::stringstream ss;
ss << "Input " << i << " type '" << inputs[i]->get_element_type() << "' does not match Parameter type '" ss << "Input " << i << " type '" << inputs[i].get_element_type() << "' does not match Parameter type '"
<< parameters[i]->get_element_type() << "'"; << parameters[i]->get_element_type() << "'";
throw runtime_error(ss.str()); throw std::runtime_error(ss.str());
}
if (!(parameters[i]->get_output_partial_shape(0).relaxes(inputs[i]->get_partial_shape()))) {
stringstream ss;
ss << "Input " << i << " shape " << inputs[i]->get_partial_shape() << " does not match Parameter shape "
<< parameters[i]->get_output_partial_shape(0);
throw runtime_error(ss.str());
} }
} }
for (size_t i = 0; i < results.size(); i++) { for (size_t i = 0; i < results.size(); i++) {
if (outputs[i]->get_element_type().is_static() && results[i]->get_element_type().is_static() && if (outputs[i].get_element_type().is_static() && results[i]->get_element_type().is_static() &&
results[i]->get_element_type() != outputs[i]->get_element_type()) { results[i]->get_element_type() != outputs[i].get_element_type()) {
stringstream ss; std::stringstream ss;
ss << "Output " << i << " type '" << outputs[i]->get_element_type() << "' does not match Result type '" ss << "Output " << i << " type '" << outputs[i].get_element_type() << "' does not match Result type '"
<< results[i]->get_element_type() << "'"; << results[i]->get_element_type() << "'";
throw runtime_error(ss.str()); throw std::runtime_error(ss.str());
}
if (!outputs[i]->get_partial_shape().relaxes(results[i]->get_output_partial_shape(0))) {
stringstream ss;
ss << "Output " << i << " shape " << outputs[i]->get_partial_shape() << " does not match max Result shape "
<< results[i]->get_output_partial_shape(0).get_max_shape();
throw runtime_error(ss.str());
} }
} }
} }
const ngraph::ParameterVector& runtime::Executable::get_parameters() const { const ov::ParameterVector& ov::runtime::Executable::get_parameters() const {
return m_parameters; return m_parameters;
} }
const ngraph::ResultVector& runtime::Executable::get_results() const { const ov::ResultVector& ov::runtime::Executable::get_results() const {
return m_results; return m_results;
} }
size_t runtime::Executable::get_preferred_pipeline_depth() const { void ov::runtime::Executable::set_parameters_and_results(const ov::Model& model) {
return 2; m_parameters = model.get_parameters();
m_results = model.get_results();
} }
void runtime::Executable::set_parameters_and_results(const Function& func) { ov::Tensor ov::runtime::Executable::create_input_tensor(size_t /* input_index */) {
m_parameters = func.get_parameters(); OPENVINO_NOT_IMPLEMENTED;
m_results = func.get_results();
} }
vector<runtime::PerformanceCounter> runtime::Executable::get_performance_data() const { ov::Tensor ov::runtime::Executable::create_output_tensor(size_t /* output_index */) {
return vector<PerformanceCounter>(); OPENVINO_NOT_IMPLEMENTED;
} }
void runtime::Executable::save(std::ostream& /* output_stream */) { std::vector<ov::Tensor> ov::runtime::Executable::create_input_tensor(size_t /* input_index */,
throw runtime_error("save operation unimplemented."); size_t /* pipeline_depth */) {
OPENVINO_NOT_IMPLEMENTED;
} }
shared_ptr<runtime::Tensor> runtime::Executable::create_input_tensor(size_t /* input_index */) { std::vector<ov::Tensor> ov::runtime::Executable::create_output_tensor(size_t /* output_index */,
throw runtime_error("create_input_tensor unimplemented"); size_t /* pipeline_depth */) {
} OPENVINO_NOT_IMPLEMENTED;
shared_ptr<runtime::Tensor> runtime::Executable::create_input_tensor(size_t /* input_index */,
void* /* memory_pointer */) {
throw runtime_error("create_input_tensor unimplemented");
}
shared_ptr<runtime::Tensor> runtime::Executable::create_output_tensor(size_t /* output_index */) {
throw runtime_error("create_output_tensor unimplemented");
}
shared_ptr<runtime::Tensor> runtime::Executable::create_output_tensor(size_t /* output_index */,
void* /* memory_pointer */) {
throw runtime_error("create_output_tensor unimplemented");
}
vector<shared_ptr<runtime::Tensor>> runtime::Executable::create_input_tensor(size_t /* input_index */,
size_t /* pipeline_depth */) {
throw runtime_error("create_input_tensor unimplemented");
}
vector<shared_ptr<runtime::Tensor>> runtime::Executable::create_input_tensor(size_t /* input_index */,
size_t /* pipeline_depth */,
std::vector<void*> /* memory_pointer */) {
throw runtime_error("create_input_tensor unimplemented");
}
vector<shared_ptr<runtime::Tensor>> runtime::Executable::create_output_tensor(size_t /* output_index */,
size_t /* pipeline_depth */) {
throw runtime_error("create_output_tensor unimplemented");
}
vector<shared_ptr<runtime::Tensor>> runtime::Executable::create_output_tensor(size_t /* output_index */,
size_t /* pipeline_depth */,
std::vector<void*> /* memory_pointer */) {
throw runtime_error("create_output_tensor unimplemented");
} }

104
src/plugins/template/backend/executable.hpp
View File

@ -6,19 +6,15 @@
#include <memory> #include <memory>
#include "ngraph/function.hpp" #include "openvino/core/model.hpp"
#include "ngraph/runtime/tensor.hpp" #include "openvino/core/node_vector.hpp"
#include "ngraph/shape.hpp" #include "openvino/op/parameter.hpp"
#include "ngraph/type/element_type.hpp" #include "openvino/runtime/tensor.hpp"
#include "performance_counter.hpp"
namespace ngraph { namespace ov {
namespace runtime { namespace runtime {
class Executable;
}
} // namespace ngraph
class ngraph::runtime::Executable { class Executable {
public: public:
Executable(); Executable();
virtual ~Executable(); virtual ~Executable();
@ -26,71 +22,38 @@ public:
/// \param outputs vector of runtime::Tensor used as outputs /// \param outputs vector of runtime::Tensor used as outputs
/// \param inputs vector of runtime::Tensor used as inputs /// \param inputs vector of runtime::Tensor used as inputs
/// \returns true if iteration is successful, false otherwise /// \returns true if iteration is successful, false otherwise
virtual bool call(const std::vector<std::shared_ptr<runtime::Tensor>>& outputs, virtual bool call(std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs) = 0;
const std::vector<std::shared_ptr<runtime::Tensor>>& inputs) = 0;
/// \brief Executes a single iteration of a Function. /// \brief Executes a single iteration of a Function.
/// \param outputs vector of runtime::Tensor used as outputs /// \param outputs vector of runtime::Tensor used as outputs
/// \param inputs vector of runtime::Tensor used as inputs /// \param inputs vector of runtime::Tensor used as inputs
/// \returns true if iteration is successful, false otherwise /// \returns true if iteration is successful, false otherwise
bool call_with_validate(const std::vector<std::shared_ptr<runtime::Tensor>>& outputs, bool call_with_validate(std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs);
const std::vector<std::shared_ptr<runtime::Tensor>>& inputs);
/// \brief Collect performance information gathered on a Function.
/// \returns Vector of PerformanceCounter information.
virtual std::vector<PerformanceCounter> get_performance_data() const;
/// \brief Validates a Function. /// \brief Validates a Function.
/// \param outputs vector of runtime::Tensor used as outputs /// \param outputs vector of runtime::Tensor used as outputs
/// \param inputs vector of runtime::Tensor used as inputs /// \param inputs vector of runtime::Tensor used as inputs
void validate(const std::vector<std::shared_ptr<runtime::Tensor>>& outputs, void validate(const std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs);
const std::vector<std::shared_ptr<runtime::Tensor>>& inputs);
/// \brief Query the input Parameters /// \brief Query the input Parameters
/// \returns an ngraph::op::ParameterVector of all input parameters /// \returns an ngraph::op::ParameterVector of all input parameters
const ngraph::ParameterVector& get_parameters() const; const ov::ParameterVector& get_parameters() const;
/// \brief Query the output Results /// \brief Query the output Results
/// \returns an ngraph::ResultVector of all input parameters /// \returns an ngraph::ResultVector of all input parameters
const ngraph::ResultVector& get_results() const; const ov::ResultVector& get_results() const;
/// \brief Get the preferred pipeline_depth for this executable
/// \returns preferred pipeline_depth
virtual size_t get_preferred_pipeline_depth() const;
/// \brief Save this compiled Executable to an output stream.
/// Saved stream may be read with Backend::load
virtual void save(std::ostream& output_stream);
/// \brief Create an input Tensor /// \brief Create an input Tensor
/// \param input_index The index position in the input Parameter vector. This would be the same /// \param input_index The index position in the input Parameter vector. This would be the same
/// order of Parameters passed into the inputs in the call() method. /// order of Parameters passed into the inputs in the call() method.
/// \returns A Tensor /// \returns A Tensor
virtual std::shared_ptr<runtime::Tensor> create_input_tensor(size_t input_index); virtual ov::Tensor create_input_tensor(size_t input_index);
/// \brief Create an input Tensor
/// \param input_index The index position in the input Parameter vector. This would be the same
/// order of Parameters passed into the inputs in the call() method.
/// \param memory_pointer A pointer to a buffer used for this tensor. The size of the buffer
/// must be sufficient to contain the tensor. The lifetime of the buffer is the
/// responsibility of the caller and must outlive the created Tensor.
/// \returns A Tensor
virtual std::shared_ptr<runtime::Tensor> create_input_tensor(size_t input_index, void* memory_pointer);
/// \brief Create an output Tensor /// \brief Create an output Tensor
/// \param output_index The index position in the output Result vector. This would be the same /// \param output_index The index position in the output Result vector. This would be the same
/// order of Results passed into the outputs in the call() method. /// order of Results passed into the outputs in the call() method.
/// \returns A Tensor /// \returns A Tensor
virtual std::shared_ptr<runtime::Tensor> create_output_tensor(size_t output_index); virtual ov::Tensor create_output_tensor(size_t output_index);
/// \brief Create an output Tensor
/// \param output_index The index position in the output Result vector. This would be the same
/// order of Results passed into the outputs in the call() method.
/// \param memory_pointer A pointer to a buffer used for this tensor. The size of the buffer
/// must be sufficient to contain the tensor. The lifetime of the buffer is the
/// responsibility of the caller and must outlive the created Tensor.
/// \returns A Tensor
virtual std::shared_ptr<runtime::Tensor> create_output_tensor(size_t output_index, void* memory_pointer);
/// \brief Create a vector of input Tensors /// \brief Create a vector of input Tensors
/// \param input_index The index position in the input Parameter vector. This would be the same /// \param input_index The index position in the input Parameter vector. This would be the same
@ -98,21 +61,7 @@ public:
/// \param pipeline_depth The number of stages in the input pipeline. For double-buffered input /// \param pipeline_depth The number of stages in the input pipeline. For double-buffered input
/// you would specify pipeline_depth=2 /// you would specify pipeline_depth=2
/// \returns A vector of Tensors, one for each stage of the pipeline /// \returns A vector of Tensors, one for each stage of the pipeline
virtual std::vector<std::shared_ptr<runtime::Tensor>> create_input_tensor(size_t input_index, virtual std::vector<ov::Tensor> create_input_tensor(size_t input_index, size_t pipeline_depth);
size_t pipeline_depth);
/// \brief Create a vector of input Tensors
/// \param input_index The index position in the input Parameter vector. This would be the same
/// order of Parameters passed into the inputs in the call() method.
/// \param pipeline_depth The number of stages in the input pipeline. For double-buffered input
/// you would specify pipeline_depth=2
/// \param memory_pointers A vector of pointers to buffers used for this tensors. The size of
/// the buffer must be sufficient to contain the tensor. The lifetime of the buffers is the
/// responsibility of the caller and must outlive the created Tensor.
/// \returns A vector of Tensors, one for each stage of the pipeline
virtual std::vector<std::shared_ptr<runtime::Tensor>> create_input_tensor(size_t input_index,
size_t pipeline_depth,
std::vector<void*> memory_pointers);
/// \brief Create a vector of output Tensors /// \brief Create a vector of output Tensors
/// \param output_index The index position in the output Result vector. This would be the same /// \param output_index The index position in the output Result vector. This would be the same
@ -120,28 +69,17 @@ public:
/// \param pipeline_depth The number of stages in the output pipeline. For double-buffered /// \param pipeline_depth The number of stages in the output pipeline. For double-buffered
/// output you would specify pipeline_depth=2 /// output you would specify pipeline_depth=2
/// \returns A vector of Tensors, one for each stage of the pipeline /// \returns A vector of Tensors, one for each stage of the pipeline
virtual std::vector<std::shared_ptr<runtime::Tensor>> create_output_tensor(size_t output_index, virtual std::vector<ov::Tensor> create_output_tensor(size_t output_index, size_t pipeline_depth);
size_t pipeline_depth);
/// \brief Create a vector of output Tensors
/// \param output_index The index position in the output Result vector. This would be the same
/// order of Results passed into the outputs in the call() method.
/// \param pipeline_depth The number of stages in the output pipeline. For double-buffered
/// output you would specify pipeline_depth=2
/// \param memory_pointers A vector of pointers to buffers used for this tensors. The size of
/// the buffer must be sufficient to contain the tensor. The lifetime of the buffers is the
/// responsibility of the caller and must outlive the created Tensor.
/// \returns A vector of Tensors, one for each stage of the pipeline
virtual std::vector<std::shared_ptr<runtime::Tensor>> create_output_tensor(size_t output_index,
size_t pipeline_depth,
std::vector<void*> memory_pointers);
protected: protected:
/// \brief Called at the end of compile to the values to be returned by get_parameters /// \brief Called at the end of compile to the values to be returned by get_parameters
/// and get_results /// and get_results
/// \param func The function with Results fully resolved. /// \param func The function with Results fully resolved.
void set_parameters_and_results(const Function& func); void set_parameters_and_results(const ov::Model& model);
ngraph::ParameterVector m_parameters; ov::ParameterVector m_parameters;
ngraph::ResultVector m_results; ov::ResultVector m_results;
}; };
} // namespace runtime
} // namespace ov

53
src/plugins/template/backend/int_backend.cpp
View File

@ -5,54 +5,27 @@
#include "int_backend.hpp" #include "int_backend.hpp"
#include "int_executable.hpp" #include "int_executable.hpp"
#include "ngraph/except.hpp"
#include "ngraph/runtime/host_tensor.hpp"
#include "ngraph/util.hpp"
using namespace std; ov::runtime::interpreter::INTBackend::INTBackend() {}
using namespace ngraph;
runtime::interpreter::INTBackend::INTBackend() {} ov::runtime::interpreter::INTBackend::INTBackend(const std::vector<std::string>& unsupported_op_name_list)
runtime::interpreter::INTBackend::INTBackend(const vector<string>& unsupported_op_name_list)
: m_unsupported_op_name_list{unsupported_op_name_list.begin(), unsupported_op_name_list.end()} {} : m_unsupported_op_name_list{unsupported_op_name_list.begin(), unsupported_op_name_list.end()} {}
shared_ptr<runtime::Tensor> runtime::interpreter::INTBackend::create_tensor() { ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor() {
return make_shared<runtime::HostTensor>(); return ov::Tensor();
} }
shared_ptr<runtime::Tensor> runtime::interpreter::INTBackend::create_tensor(const element::Type& type, ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor(const element::Type& type, const Shape& shape) {
const Shape& shape) { return ov::Tensor(type, shape);
return make_shared<runtime::HostTensor>(type, shape);
} }
shared_ptr<runtime::Tensor> runtime::interpreter::INTBackend::create_dynamic_tensor(const element::Type& type, ov::Tensor ov::runtime::interpreter::INTBackend::create_tensor(const element::Type& type,
const PartialShape& pshape) { const Shape& shape,
return make_shared<runtime::HostTensor>(type, pshape); void* memory_pointer) {
return ov::Tensor(type, shape, memory_pointer);
} }
shared_ptr<runtime::Tensor> runtime::interpreter::INTBackend::create_tensor(const element::Type& type, std::shared_ptr<ov::runtime::Executable> ov::runtime::interpreter::INTBackend::compile(
const Shape& shape, std::shared_ptr<ov::Model> model) {
void* memory_pointer) { return std::make_shared<INTExecutable>(model);
return make_shared<runtime::HostTensor>(type, shape, memory_pointer);
}
shared_ptr<runtime::Executable> runtime::interpreter::INTBackend::compile(shared_ptr<Function> function,
bool enable_performance_collection) {
return make_shared<INTExecutable>(function, enable_performance_collection);
}
bool runtime::interpreter::INTBackend::is_supported(const Node& node) const {
return m_unsupported_op_name_list.find(node.description()) == m_unsupported_op_name_list.end();
}
bool runtime::interpreter::INTBackend::set_config(const map<string, string>& config, string& error) {
bool rc = false;
auto it = config.find("test_echo");
error = "";
if (it != config.end()) {
error = it->second;
rc = true;
}
return rc;
} }

29
src/plugins/template/backend/int_backend.hpp
View File

@ -11,18 +11,13 @@
#include <vector> #include <vector>
#include "backend.hpp" #include "backend.hpp"
#include "ngraph/runtime/tensor.hpp" #include "openvino/core/model.hpp"
namespace ngraph { namespace ov {
namespace runtime { namespace runtime {
namespace interpreter { namespace interpreter {
class INTBackend;
class INTExecutable;
} // namespace interpreter
} // namespace runtime
} // namespace ngraph
class ngraph::runtime::interpreter::INTBackend : public Backend { class INTBackend : public Backend {
public: public:
INTBackend(); INTBackend();
INTBackend(const std::vector<std::string>& unsupported_op_name_list); INTBackend(const std::vector<std::string>& unsupported_op_name_list);
@ -30,20 +25,18 @@ public:
INTBackend(INTBackend&&) = delete; INTBackend(INTBackend&&) = delete;
INTBackend& operator=(const INTBackend&) = delete; INTBackend& operator=(const INTBackend&) = delete;
std::shared_ptr<Tensor> create_tensor() override; ov::Tensor create_tensor() override;
std::shared_ptr<Tensor> create_tensor(const element::Type& type, const Shape& shape, void* memory_pointer) override; ov::Tensor create_tensor(const element::Type& type, const Shape& shape, void* memory_pointer) override;
std::shared_ptr<Tensor> create_tensor(const element::Type& type, const Shape& shape) override; ov::Tensor create_tensor(const element::Type& type, const Shape& shape) override;
std::shared_ptr<Tensor> create_dynamic_tensor(const element::Type& type, const PartialShape& shape) override;
std::shared_ptr<Executable> compile(std::shared_ptr<Function> function, std::shared_ptr<Executable> compile(std::shared_ptr<ov::Model> model) override;
bool enable_performance_data = false) override;
bool is_supported(const Node& node) const override;
bool set_config(const std::map<std::string, std::string>& config, std::string& error) override;
private: private:
std::set<std::string> m_unsupported_op_name_list; std::set<std::string> m_unsupported_op_name_list;
}; };
} // namespace interpreter
} // namespace runtime
} // namespace ov

299
src/plugins/template/backend/int_executable.cpp
View File

@ -5,32 +5,85 @@
#include "int_executable.hpp" #include "int_executable.hpp"
#include <cstring> #include <cstring>
#include <limits>
#include <openvino/op/util/variable_context.hpp> #include <openvino/op/util/variable_context.hpp>
#include "evaluates_map.hpp" #include "evaluates_map.hpp"
#include "ngraph/except.hpp" #include "openvino/op/parameter.hpp"
#include "ngraph/ops.hpp" #include "openvino/op/result.hpp"
#include "ngraph/type/bfloat16.hpp" #include "openvino/op/util/op_types.hpp"
#include "ngraph/type/float16.hpp"
#include "ngraph/util.hpp"
#include "tensor_conversion_util.hpp" #include "tensor_conversion_util.hpp"
using namespace std;
using namespace ngraph;
NGRAPH_SUPPRESS_DEPRECATED_START NGRAPH_SUPPRESS_DEPRECATED_START
class TemporaryOverrideOutputs { namespace {
std::shared_ptr<Node> node;
std::vector<PartialShape> orig_shapes; class DynamicTensor : public ngraph::runtime::HostTensor {
private:
ov::Tensor tensor;
public: public:
TemporaryOverrideOutputs(std::shared_ptr<Node> node, const std::vector<std::shared_ptr<HostTensor>>& args) DynamicTensor(const ov::element::Type& type) : ngraph::runtime::HostTensor(type, ov::PartialShape::dynamic()) {}
: node(node) {
ov::Tensor get_tensor() {
return tensor;
}
protected:
void allocate_buffer() override {
OPENVINO_ASSERT(get_partial_shape().is_static(),
"Attempt to allocate buffer for tensor with partial shape: ",
get_partial_shape());
OPENVINO_ASSERT(get_element_type().is_static(),
"Attempt to allocate buffer for tensor with dynamic type: ",
get_element_type());
m_buffer_size = m_descriptor->size();
tensor = ov::Tensor(get_element_type(), get_partial_shape().get_shape());
m_memory_pointer = tensor.data();
m_aligned_buffer_pool = m_memory_pointer;
}
};
inline ngraph::HostTensorPtr make_tmp_host_tensor(const ov::Tensor& t) {
OPENVINO_SUPPRESS_DEPRECATED_START
if (!t) {
return std::make_shared<DynamicTensor>(ov::element::dynamic);
} else if (t.get_shape() == ov::Shape{0, std::numeric_limits<size_t>::max()}) {
return std::make_shared<DynamicTensor>(t.get_element_type());
} else {
return std::make_shared<ngraph::runtime::HostTensor>(t.get_element_type(), t.get_shape(), t.data());
}
OPENVINO_SUPPRESS_DEPRECATED_END
}
inline ngraph::HostTensorVector create_tmp_tensors(const ov::TensorVector& tensors) {
ngraph::HostTensorVector result;
result.reserve(tensors.size());
for (const auto& tensor : tensors) {
result.push_back(make_tmp_host_tensor(tensor));
}
return result;
}
inline void update_output_tensors(ov::TensorVector& output_values, const ngraph::HostTensorVector& outputs) {
OPENVINO_ASSERT(output_values.size() == outputs.size());
for (size_t i = 0; i < outputs.size(); i++) {
if (auto dyn_output = std::dynamic_pointer_cast<DynamicTensor>(outputs[i])) {
output_values[i] = dyn_output->get_tensor();
}
}
}
} // namespace
class TemporaryOverrideOutputs {
std::shared_ptr<ov::Node> node;
std::vector<ov::PartialShape> orig_shapes;
public:
TemporaryOverrideOutputs(std::shared_ptr<ov::Node> node, const std::vector<ov::Tensor>& args) : node(node) {
for (size_t i = 0; i < args.size(); ++i) { for (size_t i = 0; i < args.size(); ++i) {
auto output = node->get_input_source_output(i); auto output = node->get_input_source_output(i);
orig_shapes.push_back(output.get_partial_shape()); orig_shapes.push_back(output.get_partial_shape());
output.get_tensor().set_partial_shape(args[i]->get_shape()); output.get_tensor().set_partial_shape(args[i].get_shape());
} }
} }
@ -42,43 +95,23 @@ public:
} }
}; };
runtime::interpreter::INTExecutable::INTExecutable(const shared_ptr<Function>& function, ov::runtime::interpreter::INTExecutable::INTExecutable(const std::shared_ptr<ov::Model>& model) : m_is_compiled{true} {
bool enable_performance_collection) m_model = model->clone();
: m_is_compiled{true}, for (auto node : m_model->get_ordered_ops()) {
m_performance_counters_enabled{enable_performance_collection} {
m_function = clone_function(*function);
for (auto node : m_function->get_ordered_ops()) {
m_nodes.push_back(node); m_nodes.push_back(node);
} }
set_parameters_and_results(*m_function); set_parameters_and_results(*m_model);
} }
bool runtime::interpreter::INTExecutable::call(const vector<shared_ptr<runtime::Tensor>>& outputs, bool ov::runtime::interpreter::INTExecutable::call(std::vector<ov::Tensor>& outputs,
const vector<shared_ptr<runtime::Tensor>>& inputs) { const std::vector<ov::Tensor>& inputs) {
// convert inputs to HostTensor
vector<shared_ptr<HostTensor>> func_inputs;
for (const auto& tensor : inputs) {
auto host_tensor = static_pointer_cast<runtime::HostTensor>(tensor);
func_inputs.push_back(host_tensor);
}
if (m_nan_check_enabled) {
perform_nan_check(func_inputs);
}
// convert outputs to HostTensor
vector<shared_ptr<HostTensor>> func_outputs;
for (const auto& tensor : outputs) {
auto host_tensor = static_pointer_cast<runtime::HostTensor>(tensor);
func_outputs.push_back(host_tensor);
}
// map function params -> HostTensor // map function params -> HostTensor
std::unordered_map<std::shared_ptr<ov::descriptor::Tensor>, shared_ptr<HostTensor>> tensor_map; std::unordered_map<std::shared_ptr<ov::descriptor::Tensor>, ov::Tensor> tensor_map;
size_t input_count = 0; size_t input_count = 0;
for (const auto& param : get_parameters()) { for (const auto& param : get_parameters()) {
for (size_t i = 0; i < param->get_output_size(); ++i) { for (size_t i = 0; i < param->get_output_size(); ++i) {
auto tensor = param->output(i).get_tensor_ptr(); auto tensor = param->output(i).get_tensor_ptr();
tensor_map.insert({tensor, func_inputs[input_count++]}); tensor_map.insert({tensor, inputs[input_count++]});
} }
} }
@ -96,186 +129,138 @@ bool runtime::interpreter::INTExecutable::call(const vector<shared_ptr<runtime::
// for each ordered op in the graph // for each ordered op in the graph
for (const auto& op : m_nodes) { for (const auto& op : m_nodes) {
if (dynamic_pointer_cast<op::Parameter>(op) != nullptr) { if (std::dynamic_pointer_cast<ov::op::v0::Parameter>(op)) {
continue; continue;
} }
// get op inputs from map // get op inputs from map
vector<shared_ptr<HostTensor>> op_inputs; std::vector<ov::Tensor> op_inputs;
for (auto input : op->inputs()) { for (auto input : op->inputs()) {
auto tensor = input.get_tensor_ptr(); auto tensor = input.get_tensor_ptr();
op_inputs.push_back(tensor_map.at(tensor)); op_inputs.push_back(tensor_map.at(tensor));
} }
TemporaryOverrideOutputs overrider(op, op_inputs); TemporaryOverrideOutputs overrider(op, op_inputs);
OutputVector outputs; OutputVector output_ports;
for (size_t i = 0; i < op->inputs().size(); ++i) { for (size_t i = 0; i < op->inputs().size(); ++i) {
outputs.push_back(op->get_input_source_output(i)); output_ports.push_back(op->get_input_source_output(i));
} }
auto cloned_node = op->clone_with_new_inputs(outputs); auto cloned_node = op->clone_with_new_inputs(output_ports);
// get op outputs from map or create // get op outputs from map or create
vector<shared_ptr<HostTensor>> op_outputs; std::vector<ov::Tensor> op_outputs;
for (size_t i = 0; i < op->get_output_size(); ++i) { for (size_t i = 0; i < op->get_output_size(); ++i) {
auto tensor = op->output(i).get_tensor_ptr(); auto tensor = op->output(i).get_tensor_ptr();
shared_ptr<HostTensor> host_tensor; ov::Tensor host_tensor;
auto it = tensor_map.find(tensor); auto it = tensor_map.find(tensor);
if (op::is_output(op)) { auto output = cloned_node->output(i);
host_tensor = func_outputs[results_map[tensor]]; if (op::util::is_output(op) || it == tensor_map.end() || !it->second) {
} else if (it == tensor_map.end()) { host_tensor = ov::Tensor(output.get_element_type(),
// Use cloned_node to create HostTensor with static dimensions output.get_partial_shape().is_dynamic()
host_tensor = make_shared<HostTensor>(cloned_node->output(i)); ? ov::Shape{0, std::numeric_limits<size_t>::max()}
tensor_map.insert({tensor, host_tensor}); : output.get_shape());
} else { } else {
host_tensor = it->second; host_tensor = it->second;
} }
op_outputs.push_back(host_tensor); op_outputs.push_back(host_tensor);
} }
if (m_performance_counters_enabled) {
m_timer_map[op].start();
}
if (auto var_extension = std::dynamic_pointer_cast<ov::op::util::VariableExtension>(cloned_node)) { if (auto var_extension = std::dynamic_pointer_cast<ov::op::util::VariableExtension>(cloned_node)) {
auto variable = var_extension->get_variable(); auto variable = var_extension->get_variable();
if (!variable_context.get_variable_value(variable)) { if (!variable_context.get_variable_value(variable)) {
auto h_tensor = std::make_shared<ngraph::HostTensor>(cloned_node->get_input_element_type(0), auto h_tensor = ov::Tensor(cloned_node->get_input_element_type(0), cloned_node->get_input_shape(0));
cloned_node->get_input_shape(0)); // h_tensor->write(h_tensor->get_data_ptr(), h_tensor->get_size_in_bytes());
h_tensor->write(h_tensor->get_data_ptr(), h_tensor->get_size_in_bytes()); const auto tensor_input = make_tmp_host_tensor(h_tensor);
variable_context.set_variable_value(variable, std::make_shared<VariableValue>(h_tensor)); variable_context.set_variable_value(variable,
std::make_shared<ov::op::util::VariableValue>(tensor_input));
} }
} }
const auto tensor_inputs = ov::util::wrap_tensors(op_inputs);
auto tensor_outputs = ov::util::wrap_tensors(op_outputs);
// Call evaluate for cloned_node with static shapes // Call evaluate for cloned_node with static shapes
if (cloned_node->evaluate(tensor_outputs, tensor_inputs, eval_context)) { if (!cloned_node->evaluate(op_outputs, op_inputs, eval_context)) {
ov::util::update_output_host_tensors(op_outputs, tensor_outputs);
} else {
evaluate_node(cloned_node, op_outputs, op_inputs); evaluate_node(cloned_node, op_outputs, op_inputs);
} }
if (m_performance_counters_enabled) { // Update tensors in tensor map
m_timer_map[op].stop(); for (size_t i = 0; i < op->get_output_size(); ++i) {
} auto tensor = op->output(i).get_tensor_ptr();
if (m_nan_check_enabled) { tensor_map.insert({tensor, op_outputs[i]});
perform_nan_check(op_outputs, op.get()); if (op::util::is_output(op)) {
auto& output = outputs[results_map[tensor]];
if (!output || output.get_shape() != op_outputs[i].get_shape()) {
outputs[results_map[tensor]] = op_outputs[i];
} else {
op_outputs[i].copy_to(output);
}
}
} }
} }
return true; return true;
} }
vector<runtime::PerformanceCounter> runtime::interpreter::INTExecutable::get_performance_data() const { std::shared_ptr<ov::op::v0::Parameter> ov::runtime::interpreter::INTExecutable::get_parameter(size_t index) const {
vector<runtime::PerformanceCounter> rc;
for (const pair<shared_ptr<const Node>, stopwatch> p : m_timer_map) {
rc.emplace_back(p.first, p.second.get_total_microseconds(), p.second.get_call_count());
}
return rc;
}
void runtime::interpreter::INTExecutable::perform_nan_check(const vector<shared_ptr<HostTensor>>& tensors,
const Node* op) {
size_t arg_number = 1;
for (const shared_ptr<HostTensor>& tensor : tensors) {
const element::Type& type = tensor->get_element_type();
if (type == element::f32) {
const float* data = tensor->get_data_ptr<float>();
for (size_t i = 0; i < tensor->get_element_count(); i++) {
if (std::isnan(data[i])) {
if (op) {
throw runtime_error("nan found in op '" + op->get_name() + "' output");
} else {
throw runtime_error("nan found in function's input tensor number " + to_string(arg_number));
}
}
}
} else if (type == element::f64) {
const double* data = tensor->get_data_ptr<double>();
for (size_t i = 0; i < tensor->get_element_count(); i++) {
if (std::isnan(data[i])) {
if (op) {
throw runtime_error("nan found in op '" + op->get_name() + "' output");
} else {
throw runtime_error("nan found in function's input tensor number " + to_string(arg_number));
}
}
}
}
arg_number++;
}
}
shared_ptr<ngraph::op::Parameter> runtime::interpreter::INTExecutable::get_parameter(size_t index) const {
const ParameterVector& parameters = get_parameters(); const ParameterVector& parameters = get_parameters();
NGRAPH_CHECK(index < parameters.size(), "create_tensor for input out of bounds"); NGRAPH_CHECK(index < parameters.size(), "create_tensor for input out of bounds");
return parameters[index]; return parameters[index];
} }
shared_ptr<ngraph::op::Result> runtime::interpreter::INTExecutable::get_result(size_t index) const { std::shared_ptr<ov::op::v0::Result> ov::runtime::interpreter::INTExecutable::get_result(size_t index) const {
const ResultVector& results = get_results(); const ResultVector& results = get_results();
NGRAPH_CHECK(index < results.size(), "create_tensor for input out of bounds"); NGRAPH_CHECK(index < results.size(), "create_tensor for input out of bounds");
return results[index]; return results[index];
} }
shared_ptr<runtime::Tensor> runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index) { ov::Tensor ov::runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index) {
shared_ptr<op::Parameter> parameter = get_parameter(input_index); std::shared_ptr<op::v0::Parameter> parameter = get_parameter(input_index);
return make_shared<runtime::HostTensor>(parameter->get_element_type(), parameter->get_shape()); return ov::Tensor(parameter->get_element_type(), parameter->get_shape());
} }
shared_ptr<runtime::Tensor> runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index) { ov::Tensor ov::runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index) {
shared_ptr<op::Result> result = get_result(output_index); std::shared_ptr<op::v0::Result> result = get_result(output_index);
return make_shared<runtime::HostTensor>(result->get_element_type(), result->get_shape()); return ov::Tensor(result->get_element_type(), result->get_shape());
} }
vector<shared_ptr<runtime::Tensor>> runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index, std::vector<ov::Tensor> ov::runtime::interpreter::INTExecutable::create_input_tensor(size_t input_index,
size_t pipeline_depth) { size_t pipeline_depth) {
vector<shared_ptr<runtime::HostTensor>> tensors; std::vector<ov::Tensor> tensors;
shared_ptr<op::Parameter> parameter = get_parameter(input_index); std::shared_ptr<op::v0::Parameter> parameter = get_parameter(input_index);
for (size_t i = 0; i < pipeline_depth; i++) { for (size_t i = 0; i < pipeline_depth; i++) {
shared_ptr<runtime::HostTensor> tensor; ov::Tensor tensor;
auto t = make_shared<runtime::HostTensor>(parameter->get_element_type(), parameter->get_shape()); auto t = ov::Tensor(parameter->get_element_type(), parameter->get_shape());
tensor = static_pointer_cast<runtime::HostTensor>(t); tensors.push_back(t);
tensors.push_back(tensor);
} }
vector<shared_ptr<runtime::Tensor>> result_tensors; return tensors;
for (const shared_ptr<runtime::HostTensor>& tensor : tensors) {
result_tensors.push_back(tensor);
}
return result_tensors;
} }
vector<shared_ptr<runtime::Tensor>> runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index, std::vector<ov::Tensor> ov::runtime::interpreter::INTExecutable::create_output_tensor(size_t output_index,
size_t pipeline_depth) { size_t pipeline_depth) {
vector<shared_ptr<runtime::HostTensor>> tensors; std::vector<ov::Tensor> tensors;
shared_ptr<op::Result> result = get_result(output_index); std::shared_ptr<op::v0::Result> result = get_result(output_index);
for (size_t i = 0; i < pipeline_depth; i++) { for (size_t i = 0; i < pipeline_depth; i++) {
shared_ptr<runtime::HostTensor> tensor; ov::Tensor tensor;
auto t = make_shared<runtime::HostTensor>(result->get_element_type(), result->get_shape()); auto t = ov::Tensor(result->get_element_type(), result->get_shape());
tensor = static_pointer_cast<runtime::HostTensor>(t); tensors.push_back(t);
tensors.push_back(tensor);
} }
vector<shared_ptr<runtime::Tensor>> result_tensors; return tensors;
for (const shared_ptr<runtime::HostTensor>& tensor : tensors) {
result_tensors.push_back(tensor);
}
return result_tensors;
} }
bool runtime::interpreter::INTExecutable::evaluate_node(const std::shared_ptr<Node>& node, bool ov::runtime::interpreter::INTExecutable::evaluate_node(const std::shared_ptr<Node>& node,
const HostTensorVector& outputs, ov::TensorVector& outputs,
const HostTensorVector& inputs) const { const ov::TensorVector& inputs) const {
auto& map = runtime::interpreter::get_evaluators_map(); auto& map = ngraph::runtime::interpreter::get_evaluators_map();
auto it = map.find(node->get_type_info()); auto it = map.find(node->get_type_info());
bool res = false; bool res = false;
const auto tensor_inputs = create_tmp_tensors(inputs);
auto tensor_outputs = create_tmp_tensors(outputs);
if (it != map.end()) { if (it != map.end()) {
res = it->second(node, outputs, inputs); res = it->second(node, tensor_outputs, tensor_inputs);
if (!res) { if (!res) {
throw ngraph_error(std::string("Running evaluate method for OP ") + node->get_type_info().name + throw ngraph::ngraph_error(std::string("Running evaluate method for OP ") + node->get_type_info().name +
std::string(" failed!")); std::string(" failed!"));
} }
update_output_tensors(outputs, tensor_outputs);
} else { } else {
throw unsupported_op(std::string("Interpreter backend doesn't implement evaluate method for OP ") + throw ngraph::unsupported_op(std::string("Interpreter backend doesn't implement evaluate method for OP ") +
node->get_type_info().name); node->get_type_info().name);
} }
return res; return res;
} }

61
src/plugins/template/backend/int_executable.hpp
View File

@ -7,68 +7,47 @@
#include <initializer_list> #include <initializer_list>
#include <iostream> #include <iostream>
#include <memory> #include <memory>
#include <ngraph/runtime/host_tensor.hpp>
#include <sstream> #include <sstream>
#include <string> #include <string>
#include <vector> #include <vector>
#include "backend.hpp" #include "backend.hpp"
#include "ngraph/ops.hpp" #include "openvino/core/model.hpp"
#include "ngraph/runtime/aligned_buffer.hpp" #include "openvino/op/non_max_suppression.hpp"
#include "ngraph/runtime/reference/hard_sigmoid.hpp" #include "openvino/op/parameter.hpp"
#include "ngraph/runtime/reference/non_max_suppression.hpp" #include "openvino/op/result.hpp"
#include "ngraph/runtime/reference/reorg_yolo.hpp"
#include "ngraph/runtime/reference/tensor_iterator.hpp"
#include "ngraph/runtime/tensor.hpp"
namespace ngraph { namespace ov {
namespace runtime { namespace runtime {
namespace interpreter { namespace interpreter {
class INTBackend; class INTBackend;
class INTExecutable;
} // namespace interpreter
} // namespace runtime
} // namespace ngraph
class ngraph::runtime::interpreter::INTExecutable : public Executable { class INTExecutable : public Executable {
friend class INTBackend; friend class INTBackend;
public: public:
INTExecutable(const std::shared_ptr<Function>& function, bool enable_performance_collection = false); INTExecutable(const std::shared_ptr<ov::Model>& model);
bool call(const std::vector<std::shared_ptr<Tensor>>& outputs, bool call(std::vector<ov::Tensor>& outputs, const std::vector<ov::Tensor>& inputs) override;
const std::vector<std::shared_ptr<Tensor>>& inputs) override;
void set_nan_check(bool enable); ov::Tensor create_input_tensor(size_t input_index) override;
std::vector<PerformanceCounter> get_performance_data() const override; ov::Tensor create_output_tensor(size_t output_index) override;
std::shared_ptr<runtime::Tensor> create_input_tensor(size_t input_index) override; std::vector<ov::Tensor> create_input_tensor(size_t input_index, size_t pipeline_depth) override;
std::shared_ptr<runtime::Tensor> create_output_tensor(size_t output_index) override; std::vector<ov::Tensor> create_output_tensor(size_t output_index, size_t pipeline_depth) override;
std::vector<std::shared_ptr<runtime::Tensor>> create_input_tensor(size_t input_index,
size_t pipeline_depth) override;
std::vector<std::shared_ptr<runtime::Tensor>> create_output_tensor(size_t output_index,
size_t pipeline_depth) override;
protected: protected:
std::shared_ptr<ngraph::op::Parameter> get_parameter(size_t index) const; std::shared_ptr<ov::op::v0::Parameter> get_parameter(size_t index) const;
std::shared_ptr<ngraph::op::Result> get_result(size_t index) const; std::shared_ptr<ov::op::v0::Result> get_result(size_t index) const;
bool evaluate_node(const std::shared_ptr<Node>& node, bool evaluate_node(const std::shared_ptr<Node>& node,
const HostTensorVector& outputs, ov::TensorVector& outputs,
const HostTensorVector& inputs) const; const ov::TensorVector& inputs) const;
bool m_is_compiled = false; bool m_is_compiled = false;
bool m_nan_check_enabled = false; std::shared_ptr<ov::Model> m_model;
bool m_performance_counters_enabled = false;
std::shared_ptr<Function> m_function;
NGRAPH_SUPPRESS_DEPRECATED_START
std::unordered_map<std::shared_ptr<const Node>, stopwatch> m_timer_map;
NGRAPH_SUPPRESS_DEPRECATED_END
std::vector<std::shared_ptr<Node>> m_nodes; std::vector<std::shared_ptr<Node>> m_nodes;
static void perform_nan_check(const std::vector<std::shared_ptr<HostTensor>>&, const Node* op = nullptr);
struct InfoForNMS5 { struct InfoForNMS5 {
int64_t max_output_boxes_per_class; int64_t max_output_boxes_per_class;
float iou_threshold; float iou_threshold;
@ -84,6 +63,10 @@ protected:
ngraph::element::Type output_type; ngraph::element::Type output_type;
}; };
InfoForNMS5 get_info_for_nms5_eval(const op::v5::NonMaxSuppression* nms5, InfoForNMS5 get_info_for_nms5_eval(const ov::op::v5::NonMaxSuppression* nms5,
const std::vector<std::shared_ptr<HostTensor>>& inputs); const std::vector<std::shared_ptr<HostTensor>>& inputs);
}; };
} // namespace interpreter
} // namespace runtime
} // namespace ov

128
src/plugins/template/backend/pass/dyn_elimination.cpp
View File

@ -1,128 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "dyn_elimination.hpp"
#include <numeric>
#include "ngraph/builder/reshape.hpp"
#include "ngraph/op/broadcast.hpp"
#include "ngraph/op/range.hpp"
#include "ngraph/op/transpose.hpp"
#include "ngraph/pattern/matcher.hpp"
#include "ngraph/pattern/op/label.hpp"
#include "ngraph/runtime/reference/range.hpp"
#include "ngraph/slice_plan.hpp"
NGRAPH_SUPPRESS_DEPRECATED_START
using namespace std;
using namespace ngraph;
pass::DynElimination::DynElimination() : GraphRewrite() {
construct_range();
}
template <typename T>
std::shared_ptr<op::Constant> make_range_replacement(const element::Type& et,
const Shape& shape,
const std::shared_ptr<op::Constant>& start_arg,
const std::shared_ptr<op::Constant>& step_arg) {
std::vector<T> elements(shape_size(shape));
std::vector<T> start_vec = start_arg->get_vector<T>();
std::vector<T> step_vec = step_arg->get_vector<T>();
NGRAPH_CHECK(start_vec.size() == 1 && step_vec.size() == 1);
runtime::reference::range<T>(start_vec.data(), step_vec.data(), shape_size(shape), elements.data());
return make_shared<op::Constant>(et, shape, elements);
}
void pass::DynElimination::construct_range() {
auto start_arg_label = make_shared<pattern::op::Label>(element::f32, Shape{}, pattern::has_class<op::Constant>());
auto stop_arg_label = make_shared<pattern::op::Label>(element::f32, Shape{}, pattern::has_class<op::Constant>());
auto step_arg_label = make_shared<pattern::op::Label>(element::f32, Shape{}, pattern::has_class<op::Constant>());
auto range_pat = make_shared<op::Range>(start_arg_label, stop_arg_label, step_arg_label);
auto range_callback = [start_arg_label, stop_arg_label, step_arg_label](pattern::Matcher& m) {
auto pattern_map = m.get_pattern_map();
auto start_arg = static_pointer_cast<op::Constant>(pattern_map[start_arg_label]);
auto step_arg = static_pointer_cast<op::Constant>(pattern_map[step_arg_label]);
auto range_node = static_pointer_cast<op::Range>(m.get_match_root());
NGRAPH_CHECK(start_arg->get_output_partial_shape(0).rank().compatible(0) &&
step_arg->get_output_partial_shape(0).rank().compatible(0));
auto et = range_node->get_output_element_type(0);
auto shape = range_node->get_output_shape(0);
std::shared_ptr<op::Constant> replacement;
#if defined(__GNUC__) && !(__GNUC__ == 4 && __GNUC_MINOR__ == 8)
# pragma GCC diagnostic push
# pragma GCC diagnostic error "-Wswitch"
# pragma GCC diagnostic error "-Wswitch-enum"
#endif
switch (et) {
case element::Type_t::bf16:
replacement = make_range_replacement<bfloat16>(et, shape, start_arg, step_arg);
break;
case element::Type_t::f16:
replacement = make_range_replacement<float16>(et, shape, start_arg, step_arg);
break;
case element::Type_t::f32:
replacement = make_range_replacement<float>(et, shape, start_arg, step_arg);
break;
case element::Type_t::f64:
replacement = make_range_replacement<double>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i8:
replacement = make_range_replacement<int8_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i16:
replacement = make_range_replacement<int16_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i32:
replacement = make_range_replacement<int32_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i64:
replacement = make_range_replacement<int64_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::u8:
replacement = make_range_replacement<uint8_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::u16:
replacement = make_range_replacement<uint16_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::u32:
replacement = make_range_replacement<uint32_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::u64:
replacement = make_range_replacement<uint64_t>(et, shape, start_arg, step_arg);
break;
case element::Type_t::i4:
case element::Type_t::u4:
case element::Type_t::u1:
case element::Type_t::undefined:
case element::Type_t::dynamic:
case element::Type_t::boolean:
NGRAPH_CHECK(false, "Internal nGraph error: unsupported element type: ", et);
break;
}
#if defined(__GNUC__) && !(__GNUC__ == 4 && __GNUC_MINOR__ == 8)
# pragma GCC diagnostic pop
#endif
replace_node(range_node, replacement);
return true;
};
auto range_matcher = make_shared<pattern::Matcher>(range_pat, "DynElimination.Range");
NGRAPH_SUPPRESS_DEPRECATED_START
add_matcher(range_matcher, range_callback, all_pass_property_off);
NGRAPH_SUPPRESS_DEPRECATED_END
}

47
src/plugins/template/backend/pass/dyn_elimination.hpp
View File

@ -1,47 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "ngraph/pass/graph_rewrite.hpp"
#include "ngraph/util.hpp"
namespace ngraph {
namespace pass {
/// \brief The DynElimination pass finds dynamic operations in a graph whose
/// shape relevant inputs have already been resolved to static values, and
/// replaces those dynamic operations with the equivalent operations using
/// static inputs and attributes.
/// \details This pass should be executed after the ConstantFolding pass.
///
/// The ConstantFolding and DynElimination passes are used together to transform
/// dynamic operations in a computation graph to static operations when the
/// graph is executed with input data.
///
/// In the example shown below, the original graph is constructed with dynamic
/// broadcast operation. When the graph is executed with input data, the input
/// shapes become available, by applying the ConstantFolding and DynElimination
/// pass, the graph is updated with dynamic broadcast being replaced by a static
/// broadcast operation.
/// <table>
/// <tr>
/// <th>Original</th>
/// <th>After %ConstantFolding</th>
/// <th>After %DynElimination</th>
/// </tr>
/// <tr>
/// <td> \image html dyn_broadcast_pre_constfld.svg </td>
/// <td> \image html dyn_broadcast_post_constfld.svg </td>
/// <td> \image html dyn_broadcast_post_dyneliminate.svg </td>
/// </tr>
/// </table>
class DynElimination : public GraphRewrite {
public:
DynElimination();
private:
void construct_range();
};
} // namespace pass
} // namespace ngraph

98
src/plugins/template/backend/pass/shape_relevance.cpp
View File

@ -1,98 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "pass/shape_relevance.hpp"
#include "ngraph/graph_util.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/op/util/op_types.hpp"
using namespace ngraph;
//
// This pass refreshes the "is_relevant_to_shape" flag on each parameter. A parameter will be
// flagged as relevant to shapes if there is any path from that parameter to a shape-relevant
// input that does _not_ pass through a value-irrelevant input. For example:
//
// N0[Parameter] N1[Parameter]
// | |
// | |
// | |
// N2[v1::Reshape]
//
// N1 (but not N0) will be flagged as shape-relevant, because N1 feeds into the "shape" input
// of N2.
//
// N0[Parameter] N1[Parameter]
// | |
// | N2[ShapeOf]
// | |
// N3[v1::Reshape]
//
// Neither N0 nor N1 will be flagged as shape-relevant. (N1 does feed into the "shape" input of N3,
// but only via the value-irrelevant input of ShapeOf.)
//
bool pass::ShapeRelevance::run_on_model(const std::shared_ptr<Function>& f) {
// TODO(amprocte): We are probably reinventing the wheel with the graph traversal here; the
// reason is that we need to cut the traversal short in cases where input values are
// irrelevant. See if there is a way to reduce this duplication.
// Set of nodes that must be evaluated to determine the value of shape-relevant inputs.
std::set<Node*> shape_determinants;
// Step 1: Find root nodes (these are nodes with an output connected to a shape-relevant
// input).
for (auto& n : f->get_ops()) {
for (auto& output : n->outputs()) {
for (auto& input : output.get_target_inputs()) {
if (input.get_is_relevant_to_shapes()) {
shape_determinants.insert(n.get());
break;
}
}
}
}
// Step 2: Find all shape determinants. This is the transitive closure of R, where n1 R n2
// iff there is a data flow edge from n2 to n1 and that data flow edge is not
// value-irrelevant.
bool changes_made = false;
{
std::list<Node*> to_visit{shape_determinants.begin(), shape_determinants.end()};
std::set<Node*> already_visited;
while (!to_visit.empty()) {
auto node = to_visit.front();
to_visit.pop_front();
if (already_visited.count(node) > 0) {
continue;
}
shape_determinants.insert(node);
already_visited.insert(node);
if (op::is_parameter(node)) {
auto node_as_param = static_cast<op::Parameter*>(node);
if (!node_as_param->is_relevant_to_shapes()) {
node_as_param->set_is_relevant_to_shapes(true);
changes_made = true;
}
}
for (size_t i = 0; i < node->get_input_size(); i++) {
if (!node->input(i).get_is_relevant_to_values()) {
continue;
}
auto source_node = node->get_input_node_ptr(i);
if (already_visited.count(source_node) == 0) {
to_visit.push_front(source_node);
}
}
}
}
return changes_made;
}

17
src/plugins/template/backend/pass/shape_relevance.hpp
View File

@ -1,17 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "ngraph/pass/pass.hpp"
namespace ngraph {
namespace pass {
class ShapeRelevance : public FunctionPass {
public:
ShapeRelevance() : FunctionPass() {}
bool run_on_model(const std::shared_ptr<ngraph::Function>& m) override;
};
} // namespace pass
} // namespace ngraph

37
src/plugins/template/backend/performance_counter.hpp
View File

@ -1,37 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <cstddef>
#include <string>
#include "ngraph/node.hpp"
namespace ngraph {
namespace runtime {
class PerformanceCounter {
public:
PerformanceCounter(const std::shared_ptr<const Node>& n, size_t us, size_t calls)
: m_node(n),
m_total_microseconds(us),
m_call_count(calls) {}
std::shared_ptr<const Node> get_node() const {
return m_node;
}
size_t total_microseconds() const {
return m_total_microseconds;
}
size_t microseconds() const {
return m_call_count == 0 ? 0 : m_total_microseconds / m_call_count;
}
size_t call_count() const {
return m_call_count;
}
std::shared_ptr<const Node> m_node;
size_t m_total_microseconds;
size_t m_call_count;
};
} // namespace runtime
} // namespace ngraph

2
src/plugins/template/src/plugin.cpp
View File

@ -30,7 +30,7 @@ ov::template_plugin::Plugin::Plugin() {
set_device_name("TEMPLATE"); set_device_name("TEMPLATE");
// create ngraph backend which performs inference using ngraph reference implementations // create ngraph backend which performs inference using ngraph reference implementations
_backend = ngraph::runtime::Backend::create(); _backend = ov::runtime::Backend::create();
// create default stream executor with a given name // create default stream executor with a given name
_waitExecutor = get_executor_manager()->get_idle_cpu_streams_executor({wait_executor_name}); _waitExecutor = get_executor_manager()->get_idle_cpu_streams_executor({wait_executor_name});

2
src/plugins/template/src/plugin.hpp
View File

@ -50,7 +50,7 @@ private:
friend class CompiledModel; friend class CompiledModel;
friend class InferRequest; friend class InferRequest;
std::shared_ptr<ngraph::runtime::Backend> _backend; std::shared_ptr<ov::runtime::Backend> _backend;
Configuration _cfg; Configuration _cfg;
std::shared_ptr<ov::threading::ITaskExecutor> _waitExecutor; std::shared_ptr<ov::threading::ITaskExecutor> _waitExecutor;
}; };

23
src/plugins/template/src/sync_infer_request.cpp
View File

@ -131,23 +131,7 @@ void ov::template_plugin::InferRequest::infer_preprocess() {
get_template_model()->get_template_plugin()->_backend->create_tensor(tensor.get_element_type(), get_template_model()->get_template_plugin()->_backend->create_tensor(tensor.get_element_type(),
tensor.get_shape()); tensor.get_shape());
auto* src_data = static_cast<uint8_t*>(tensor.data()); auto* src_data = static_cast<uint8_t*>(tensor.data());
auto dst_tensor = std::dynamic_pointer_cast<ngraph::runtime::HostTensor>(m_backend_input_tensors[i]); tensor.copy_to(m_backend_input_tensors[i]);
OPENVINO_ASSERT(dst_tensor, "Template plugin error: Can't cast created tensor to HostTensor");
auto* dst_data = dst_tensor->get_data_ptr<uint8_t>();
std::vector<size_t> indexes(shape.size());
for (size_t dst_idx = 0; dst_idx < ov::shape_size(shape); dst_idx++) {
size_t val = dst_idx;
size_t src_idx = 0;
for (size_t j1 = 0; j1 < indexes.size(); j1++) {
size_t j = indexes.size() - j1 - 1;
indexes[j] = val % shape[j];
val /= shape[j];
src_idx += indexes[j] * tensor.get_strides()[j];
}
memcpy(dst_data + dst_idx * tensor.get_element_type().size(),
src_data + src_idx,
tensor.get_element_type().size());
}
} }
} }
// Tensors can be dynamic, so in this case we need to allocate tensors with right shape // Tensors can be dynamic, so in this case we need to allocate tensors with right shape
@ -196,9 +180,8 @@ void ov::template_plugin::InferRequest::infer_postprocess() {
auto host_tensor = m_backend_output_tensors[i]; auto host_tensor = m_backend_output_tensors[i];
ov::Output<const ov::Node> output{result->output(0).get_node(), result->output(0).get_index()}; ov::Output<const ov::Node> output{result->output(0).get_node(), result->output(0).get_index()};
allocate_tensor(output, [host_tensor](ov::Tensor& tensor) { allocate_tensor(output, [host_tensor](ov::Tensor& tensor) {
allocate_tensor_impl(tensor, host_tensor->get_element_type(), host_tensor->get_shape()); allocate_tensor_impl(tensor, host_tensor.get_element_type(), host_tensor.get_shape());
// tensor.set_shape(host_tensor->get_shape()); host_tensor.copy_to(tensor);
host_tensor->read(static_cast<char*>(tensor.data()), host_tensor->get_size_in_bytes());
}); });
} }
} }

6
src/plugins/template/src/sync_infer_request.hpp
View File

@ -49,9 +49,9 @@ private:
// for performance counters // for performance counters
std::array<std::chrono::duration<float, std::micro>, numOfStages> m_durations; std::array<std::chrono::duration<float, std::micro>, numOfStages> m_durations;
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> m_backend_input_tensors; std::vector<ov::Tensor> m_backend_input_tensors;
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> m_backend_output_tensors; std::vector<ov::Tensor> m_backend_output_tensors;
std::shared_ptr<ngraph::runtime::Executable> m_executable; std::shared_ptr<ov::runtime::Executable> m_executable;
}; };
// ! [infer_request:header] // ! [infer_request:header]

40
src/tests/ngraph_helpers/ngraph_functions/src/utils/ngraph_helpers.cpp
View File

@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0 // SPDX-License-Identifier: Apache-2.0
// //
#include <cstring>
#include <vector> #include <vector>
#include <memory> #include <memory>
#include <queue> #include <queue>
@ -16,6 +17,7 @@
#include <ngraph_functions/utils/ngraph_helpers.hpp> #include <ngraph_functions/utils/ngraph_helpers.hpp>
#include <ngraph/opsets/opset.hpp> #include <ngraph/opsets/opset.hpp>
#include <backend.hpp> #include <backend.hpp>
#include "openvino/runtime/tensor.hpp"
namespace ngraph { namespace ngraph {
namespace helpers { namespace helpers {
@ -84,7 +86,7 @@ std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>
interpreterFunction(const std::shared_ptr<Function> &function, interpreterFunction(const std::shared_ptr<Function> &function,
const std::vector<std::vector<std::uint8_t>> &inputs, const std::vector<std::vector<std::uint8_t>> &inputs,
const std::vector<ngraph::element::Type> &inputTypes) { const std::vector<ngraph::element::Type> &inputTypes) {
auto backend = runtime::Backend::create(); auto backend = ov::runtime::Backend::create();
const auto &parameters = function->get_parameters(); const auto &parameters = function->get_parameters();
const auto &parametersNumber = parameters.size(); const auto &parametersNumber = parameters.size();
@ -98,7 +100,7 @@ std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>
inputTypes.size(), " types"); inputTypes.size(), " types");
} }
auto inputTensors = std::vector<std::shared_ptr<runtime::Tensor>>{}; ov::TensorVector inputTensors(parametersNumber);
for (size_t i = 0; i < parametersNumber; ++i) { for (size_t i = 0; i < parametersNumber; ++i) {
const auto &parameter = parameters[i]; const auto &parameter = parameters[i];
const auto &parameterIndex = function->get_parameter_index(parameter); const auto &parameterIndex = function->get_parameter_index(parameter);
@ -120,14 +122,14 @@ std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>
" has ", inputSize, " bytes"); " has ", inputSize, " bytes");
auto tensor = backend->create_tensor(parameterType, parameterShape); auto tensor = backend->create_tensor(parameterType, parameterShape);
tensor->write(input.data(), parameterSize); std::memcpy(tensor.data(), input.data(), parameterSize);
inputTensors.push_back(tensor); inputTensors[i] = tensor;
} }
auto outputTensors = std::vector<std::shared_ptr<runtime::Tensor>>{};
const auto &results = function->get_results(); const auto &results = function->get_results();
ov::TensorVector outputTensors(results.size());
for (size_t i = 0; i < results.size(); ++i) { for (size_t i = 0; i < results.size(); ++i) {
outputTensors.push_back(std::make_shared<HostTensor>()); outputTensors[i] = ov::Tensor(results[i]->get_element_type(), {0});
} }
auto handle = backend->compile(function); auto handle = backend->compile(function);
@ -137,16 +139,16 @@ std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>
auto& output = outputs[resultIndex]; auto& output = outputs[resultIndex];
output.first = results[resultIndex]->get_element_type(); output.first = results[resultIndex]->get_element_type();
const auto& outputTensor = outputTensors[resultIndex]; const auto& outputTensor = outputTensors[resultIndex];
output.second.resize((shape_size(outputTensor->get_shape()) * outputTensor->get_element_type().bitwidth() + 7) >> 3); output.second.resize((shape_size(outputTensor.get_shape()) * outputTensor.get_element_type().bitwidth() + 7) >> 3);
outputTensors[resultIndex]->read(output.second.data(), output.second.size()); std::memcpy(output.second.data(), outputTensors[resultIndex].data(), output.second.size());
} }
return outputs; return outputs;
} }
std::vector<ov::Tensor> interpretFunction(const std::shared_ptr<Function> &function, std::vector<ov::Tensor> interpretFunction(const std::shared_ptr<Function> &function,
const std::map<std::shared_ptr<ov::Node>, ov::Tensor>& inputs) { const std::map<std::shared_ptr<ov::Node>, ov::Tensor>& inputs) {
auto backend = runtime::Backend::create(); auto backend = ov::runtime::Backend::create();
const auto &funcInputs = function->inputs(); const auto &funcInputs = function->inputs();
const auto &funcInputsNumber = funcInputs.size(); const auto &funcInputsNumber = funcInputs.size();
@ -155,7 +157,7 @@ std::vector<ov::Tensor> interpretFunction(const std::shared_ptr<Function> &funct
"Got function (", function->get_friendly_name(), ") with ", funcInputsNumber, " parameters, but ", "Got function (", function->get_friendly_name(), ") with ", funcInputsNumber, " parameters, but ",
inputsNumber, " input blobs"); inputsNumber, " input blobs");
auto inputTensors = std::vector<std::shared_ptr<runtime::Tensor>>{}; ov::TensorVector inputTensors(funcInputsNumber);
for (size_t i = 0; i < funcInputsNumber; ++i) { for (size_t i = 0; i < funcInputsNumber; ++i) {
const auto &input = funcInputs[i]; const auto &input = funcInputs[i];
const auto &inputShape = input.get_shape(); const auto &inputShape = input.get_shape();
@ -178,26 +180,20 @@ std::vector<ov::Tensor> interpretFunction(const std::shared_ptr<Function> &funct
" has ", inputTensorSize, " bytes"); " has ", inputTensorSize, " bytes");
auto tensor = backend->create_tensor(inputType, inputShape); auto tensor = backend->create_tensor(inputType, inputShape);
tensor->write(inputTensor.data(), inputSize); inputTensor.copy_to(tensor);
inputTensors.push_back(tensor); inputTensors[i] = tensor;
} }
std::vector<std::shared_ptr<runtime::Tensor>> outputTensors;
const auto &results = function->get_results(); const auto &results = function->get_results();
ov::TensorVector outputTensors(results.size());
for (size_t i = 0; i < results.size(); ++i) { for (size_t i = 0; i < results.size(); ++i) {
outputTensors.push_back(std::make_shared<HostTensor>()); outputTensors[i] = ov::Tensor(results[i]->get_element_type(), {0});
} }
auto handle = backend->compile(function); auto handle = backend->compile(function);
handle->call_with_validate(outputTensors, inputTensors); handle->call_with_validate(outputTensors, inputTensors);
std::vector<ov::Tensor> outputs;
for (const auto& outTensor : outputTensors) {
ov::Tensor tmpBuffer(outTensor->get_element_type(), outTensor->get_shape());
outTensor->read(tmpBuffer.data(), tmpBuffer.get_byte_size());
outputs.push_back(tmpBuffer);
}
return outputs; return outputTensors;
} }
std::shared_ptr<Function> foldFunction(const std::shared_ptr<Function> &function, std::shared_ptr<Function> foldFunction(const std::shared_ptr<Function> &function,