IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GPU] Dynamic kernels infra and Permute ref kernel (#13661)

Browse Source
This commit is contained in:
Vladimir Paramuzov authored and GitHub committed 2022-11-16 09:29:12 +04:00
1 parent de1a08c076
commit 21ead7cbc6
31 files changed
+655 -126
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/plugins/intel_gpu/CMakeLists.txt
+1 -1
View File
@@ -25,7 +25,7 @@ if(ENABLE_GPU_DEBUG_CAPS)
add_definitions(-DGPU_DEBUG_CONFIG=1)
endif()
set(INTEL_GPU_TARGET_OCL_VERSION "200" CACHE "Target version of OpenCL which should be used by GPU plugin" STRING)
set(INTEL_GPU_TARGET_OCL_VERSION "200" CACHE STRING "Target version of OpenCL which should be used by GPU plugin")
add_definitions(-DCL_TARGET_OPENCL_VERSION=${INTEL_GPU_TARGET_OCL_VERSION})
src/plugins/intel_gpu/include/intel_gpu/primitives/implementation_desc.hpp
+33
View File
@@ -49,6 +49,39 @@ inline std::ostream& operator<<(std::ostream& out, const impl_types& impl_type)
return out;
}
/// @brief Possible supported shape types.
enum class shape_types : uint8_t {
static_shape = 1 << 0,
dynamic_shape = 1 << 1,
any = 0xFF,
};
inline shape_types operator&(shape_types a, shape_types b) {
typedef std::underlying_type<shape_types>::type underlying_type;
return static_cast<shape_types>(static_cast<underlying_type>(a) & static_cast<underlying_type>(b));
}
inline shape_types operator|(shape_types a, shape_types b) {
typedef std::underlying_type<shape_types>::type underlying_type;
return static_cast<shape_types>(static_cast<underlying_type>(a) | static_cast<underlying_type>(b));
}
inline shape_types operator~(shape_types a) {
typedef std::underlying_type<shape_types>::type underlying_type;
return static_cast<shape_types>(~static_cast<underlying_type>(a));
}
inline std::ostream& operator<<(std::ostream& out, const shape_types& shape_type) {
switch (shape_type) {
case shape_types::static_shape: out << "static_shape"; break;
case shape_types::dynamic_shape: out << "dynamic_shape"; break;
case shape_types::any: out << "any"; break;
default: out << "unknown"; break;
}
return out;
}
/// @brief Description of primitives implementation.
struct implementation_desc {
format::type output_format; ///< Output format.
src/plugins/intel_gpu/include/intel_gpu/runtime/kernel_args.hpp
+3 -1
View File
@@ -74,7 +74,8 @@ struct argument_desc {
WEIGHTS_ZERO_POINTS,
ACTIVATIONS_ZERO_POINTS,
COMPENSATION,
INPUT_OF_FUSED_PRIMITIVE
INPUT_OF_FUSED_PRIMITIVE,
SHAPE_INFO
};
Types t;
@@ -108,6 +109,7 @@ struct kernel_arguments_data {
memory::cptr lookup_table;
memory::cptr scale_table;
memory::cptr slope;
memory::cptr shape_info;
std::vector<memory::cptr> fused_op_inputs;
int32_t split = 0;
src/plugins/intel_gpu/include/intel_gpu/runtime/layout.hpp
+2
View File
@@ -501,6 +501,8 @@ struct layout {
// for smaller buffer which, currently, should always be performed
bool identical(const layout& other) const;
ov::PartialShape transform(cldnn::format new_fmt) const;
private:
/// The size of the @ref memory (excluding padding)
ov::PartialShape size;
src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp
+4
View File
@@ -187,6 +187,10 @@ void add_required_reorders::run(program& p) {
};
}
if (original_layout.is_dynamic() && usr->type()->does_dynamic_implementation_exist(*usr)) {
correct_layout_selected = true;
}
if (usr->get_preferred_impl_type() == impl_types::onednn) {
usr->set_preferred_impl_type(impl_types::ocl);
usr->set_output_layout(original_layout, false);
src/plugins/intel_gpu/src/graph/graph_optimizer/compile_graph.cpp
+4 -1
View File
@@ -33,7 +33,10 @@ void compile_graph::run(program& p) {
std::exception_ptr exception;
for (size_t idx = 0; idx < proc_order.size(); idx++) {
auto& node = *(std::next(proc_order.begin(), idx));
if (!node->is_type<data>() && !(node->is_type<mutable_data>() && node->get_dependencies().empty()) && !node->is_dynamic()) {
bool can_select_impl = !node->is_type<data>() &&
!(node->is_type<mutable_data>() && node->get_dependencies().empty()) &&
(!node->is_dynamic() || node->type()->does_dynamic_implementation_exist(*node));
if (can_select_impl) {
tasks.push_back([node, &p, &exception] {
try {
node->selected_impl = node->type()->choose_impl(*node);
src/plugins/intel_gpu/src/graph/impls/implementation_map.hpp
+27 -11
View File
@@ -127,15 +127,18 @@ public:
using key_builder = implementation_key<primitive_kind>;
using key_type = typename key_builder::type;
using factory_type = std::function<std::unique_ptr<primitive_impl>(const typed_program_node<primitive_kind>&, const kernel_impl_params&)>;
using map_type = singleton_map<impl_types, std::pair<std::set<key_type>, factory_type>>;
using map_type = singleton_map<std::pair<impl_types, shape_types>, std::pair<std::set<key_type>, factory_type>>;
static factory_type get(const kernel_impl_params& impl_params, impl_types preferred_impl_type) {
static factory_type get(const kernel_impl_params& impl_params, impl_types preferred_impl_type, shape_types target_shape_type) {
auto input_layout = !impl_params.input_layouts.empty() ? impl_params.input_layouts[0] : layout{ov::PartialShape{}, data_types::f32, format::any};
auto key = key_builder()(input_layout);
for (auto& kv : map_type::instance()) {
impl_types impl_type = kv.first;
impl_types impl_type = kv.first.first;
shape_types supported_shape_type = kv.first.second;
if ((preferred_impl_type & impl_type) != impl_type)
continue;
if ((target_shape_type & supported_shape_type) != target_shape_type)
continue;
std::set<key_type>& keys_set = kv.second.first;
auto& factory = kv.second.second;
if (keys_set.empty() || keys_set.find(key) != keys_set.end()) {
@@ -144,28 +147,31 @@ public:
}
OPENVINO_ASSERT(false, "[GPU] implementation_map for ", typeid(primitive_kind).name(),
" could not find any implementation to match key: ", get_key_name(key),
", impl_type: ", preferred_impl_type, ", node_id: ", impl_params.desc->id);
", impl_type: ", preferred_impl_type, ", shape_type: ", target_shape_type, ", node_id: ", impl_params.desc->id);
}
// check if for a given engine and type there exist an implementation
static bool check(const kernel_impl_params& impl_params, impl_types target_impl_type) {
static bool check(const kernel_impl_params& impl_params, impl_types target_impl_type, shape_types shape_type) {
auto input_layout = !impl_params.input_layouts.empty() ? impl_params.input_layouts[0] : layout{ov::PartialShape{}, data_types::f32, format::any};
auto key = key_builder()(input_layout);
return check_key(target_impl_type, key);
return check_key(target_impl_type, key, shape_type);
}
// check if there exists a kernel implementation of a primitive with output set it primitive's output layout
static bool check_io_eq(const kernel_impl_params& impl_params, impl_types target_impl_type) {
static bool check_io_eq(const kernel_impl_params& impl_params, impl_types target_impl_type, shape_types shape_type) {
auto output_layout = !impl_params.output_layouts.empty() ? impl_params.get_output_layout() : layout{ov::PartialShape{}, data_types::f32, format::any};
auto key = key_builder()(output_layout);
return check_key(target_impl_type, key);
return check_key(target_impl_type, key, shape_type);
}
static bool check_key(impl_types target_impl_type, key_type key) {
static bool check_key(impl_types target_impl_type, key_type key, shape_types target_shape_type) {
for (auto& kv : map_type::instance()) {
impl_types impl_type = kv.first;
impl_types impl_type = kv.first.first;
shape_types supported_shape_type = kv.first.second;
if ((target_impl_type & impl_type) != impl_type)
continue;
if ((target_shape_type & supported_shape_type) != target_shape_type)
continue;
std::set<key_type>& keys_set = kv.second.first;
if (keys_set.empty())
return true;
@@ -174,6 +180,11 @@ public:
return false;
}
static void add(impl_types impl_type, shape_types shape_type, factory_type factory,
const std::vector<data_types>& types, const std::vector<format::type>& formats) {
add(impl_type, shape_type, factory, combine(types, formats));
}
static void add(impl_types impl_type, factory_type factory,
const std::vector<data_types>& types, const std::vector<format::type>& formats) {
add(impl_type, factory, combine(types, formats));
@@ -181,7 +192,12 @@ public:
static void add(impl_types impl_type, factory_type factory, std::set<key_type> keys) {
OPENVINO_ASSERT(impl_type != impl_types::any, "[GPU] Can't register impl with type any");
map_type::instance().insert({impl_type, {keys, factory}});
add(impl_type, shape_types::static_shape, factory, keys);
}
static void add(impl_types impl_type, shape_types shape_type, factory_type factory, std::set<key_type> keys) {
OPENVINO_ASSERT(impl_type != impl_types::any, "[GPU] Can't register impl with type any");
map_type::instance().insert({{impl_type, shape_type}, {keys, factory}});
}
private:
src/plugins/intel_gpu/src/graph/impls/ocl/permute.cpp
+6 -1
View File
@@ -65,12 +65,17 @@ struct permute_impl : typed_primitive_impl_ocl<permute> {
return {params, optional_params};
}
void update_dispatch_data(const kernel_impl_params& impl_param) override {
auto kernel_params = get_kernel_params(impl_param);
(_kernel_data.update_dispatch_data_func)(kernel_params.first, _kernel_data);
}
};
namespace detail {
attach_permute_impl::attach_permute_impl() {
implementation_map<permute>::add(impl_types::ocl, typed_primitive_impl_ocl<permute>::create<permute_impl>, {});
implementation_map<permute>::add(impl_types::ocl, shape_types::any, typed_primitive_impl_ocl<permute>::create<permute_impl>, {});
}
} // namespace detail
src/plugins/intel_gpu/src/graph/impls/ocl/primitive_base.hpp
+3 -1
View File
@@ -44,7 +44,7 @@ struct typed_primitive_impl_ocl : public typed_primitive_impl<PType> {
}
typed_primitive_impl_ocl(const typed_primitive_impl_ocl<PType>& other)
: typed_primitive_impl<PType>(other._weights_reorder_params, other._kernel_name)
: typed_primitive_impl<PType>(other._weights_reorder_params, other._kernel_name, other._is_dynamic)
, _node_id(other._node_id)
, _kernel_data(other._kernel_data)
, _kernel_ids(other._kernel_ids)
@@ -116,6 +116,8 @@ protected:
args.outputs.push_back(instance.output_memory_ptr(i));
}
args.shape_info = instance.shape_info_memory_ptr();
return args;
}
src/plugins/intel_gpu/src/graph/impls/ocl/roi_pooling.cpp
-1
View File
@@ -65,7 +65,6 @@ protected:
public:
static kernel_params_t get_kernel_params(const kernel_impl_params& impl_param) {
const auto& primitive = impl_param.typed_desc<roi_pooling>();
const auto& input_layout = impl_param.get_input_layout(0);
const auto& rois_layout = impl_param.get_input_layout(1);
auto params = get_default_params<kernel_selector::roi_pooling_params>(impl_param);
src/plugins/intel_gpu/src/graph/include/primitive_inst.h
+17 -2
View File
@@ -45,8 +45,8 @@ class typed_primitive_inst;
*/
struct primitive_impl {
primitive_impl() = default;
explicit primitive_impl(const kernel_selector::weights_reorder_params& params, std::string kernel_name = "")
: _weights_reorder_params(params), _kernel_name(kernel_name) {}
explicit primitive_impl(const kernel_selector::weights_reorder_params& params, std::string kernel_name = "", bool is_dynamic = false)
: _weights_reorder_params(params), _kernel_name(kernel_name), _is_dynamic(is_dynamic) {}
virtual ~primitive_impl() = default;
virtual std::vector<layout> get_internal_buffer_layouts() const = 0;
@@ -76,8 +76,17 @@ struct primitive_impl {
// If this flag is set as false, the memory allocated for this primitive is not allowed to be reused
bool can_reuse_memory = true;
void set_dynamic(bool val) { _is_dynamic = val; }
bool is_dynamic() const { return _is_dynamic; }
virtual void update_dispatch_data(const kernel_impl_params& impl_params) {
OPENVINO_ASSERT(_is_dynamic, "[GPU] update_dispatch_data is called for static shape implementation ", _kernel_name);
OPENVINO_ASSERT(false, "[GPU] update_dispatch_data is not implemented for dynamic implemenation ", _kernel_name);
}
protected:
std::string _kernel_name;
bool _is_dynamic = false;
};
/*
@@ -154,6 +163,8 @@ public:
return dep_memory_ptr(index);
}
memory::ptr shape_info_memory_ptr() const { return _shape_info_memory; }
event::ptr execute(const std::vector<event::ptr>& events);
void init_kernels(const kernels_cache& kernels_cache) {
_impl->init_kernels(kernels_cache);
@@ -229,6 +240,7 @@ protected:
std::unique_ptr<kernel_impl_params> _impl_params;
std::unique_ptr<primitive_impl> _impl;
std::unique_ptr<primitive_impl> _dynamic_impl = nullptr;
// this is a set of dependencies in terms of memory, if execution of this primitive requires data from another one,
// it should be added to this set
@@ -258,6 +270,9 @@ protected:
std::vector<memory::cptr> _intermediates_memory;
// Buffer to store actual shapes of dynamic tensor which is automatically asigned as 1st argument to shape agnostic kernels
memory::ptr _shape_info_memory = nullptr;
bool _output_changed; // todo: implement output reuse if neither of inputs has changed
bool _shape_changed = false;
bool _has_valid_input =
src/plugins/intel_gpu/src/graph/include/primitive_type.h
+3
View File
@@ -39,6 +39,9 @@ struct primitive_type {
virtual bool does_possible_implementation_exist(const program_node& node) const = 0;
virtual bool does_possible_implementation_exist(const program_node& node, const kernel_impl_params& params) const = 0;
virtual bool does_dynamic_implementation_exist(const program_node& node) const = 0;
virtual bool does_dynamic_implementation_exist(const program_node& node, const kernel_impl_params& params) const = 0;
virtual layout calc_output_layout(const program_node& node, const kernel_impl_params& params) const = 0;
virtual std::vector<layout> calc_output_layouts(const program_node& node, const kernel_impl_params& impl_param) const = 0;
virtual kernel_impl_params get_fake_aligned_params(kernel_impl_params const& orig_impl_param) const = 0;
src/plugins/intel_gpu/src/graph/include/primitive_type_base.h
+28 -4
View File
@@ -43,8 +43,10 @@ struct primitive_type_base : primitive_type {
std::unique_ptr<primitive_impl> choose_impl(const cldnn::program_node& node, const kernel_impl_params& runtime_params) const override {
OPENVINO_ASSERT(node.type() == this, "[GPU] primitive_type_base::choose_impl: primitive type mismatch");
auto factory = implementation_map<PType>::get(runtime_params, node.get_preferred_impl_type());
return factory(node, runtime_params);
auto factory = implementation_map<PType>::get(runtime_params, node.get_preferred_impl_type(), get_shape_type(runtime_params));
auto impl = factory(node, runtime_params);
impl->set_dynamic(get_shape_type(runtime_params) == shape_types::dynamic_shape);
return impl;
}
bool does_an_implementation_exist(const cldnn::program_node& node) const override {
@@ -53,7 +55,8 @@ struct primitive_type_base : primitive_type {
bool does_an_implementation_exist(const cldnn::program_node& node, const kernel_impl_params& impl_param) const override {
OPENVINO_ASSERT(node.type() == this, "[GPU] primitive_type_base::does_an_implementation_exist: primitive type mismatch");
return implementation_map<PType>::check(impl_param, node.get_preferred_impl_type());
return implementation_map<PType>::check(impl_param, node.get_preferred_impl_type(), shape_types::static_shape);
}
bool does_possible_implementation_exist(const cldnn::program_node& node) const override {
@@ -62,7 +65,16 @@ struct primitive_type_base : primitive_type {
bool does_possible_implementation_exist(const cldnn::program_node& node, const kernel_impl_params& impl_param) const override {
OPENVINO_ASSERT(node.type() == this, "[GPU] primitive_type_base::does_possible_implementation_exist: primitive type mismatch");
return implementation_map<PType>::check_io_eq(impl_param, node.get_preferred_impl_type());
return implementation_map<PType>::check_io_eq(impl_param, node.get_preferred_impl_type(), shape_types::static_shape);
}
bool does_dynamic_implementation_exist(const cldnn::program_node& node) const override {
return does_dynamic_implementation_exist(node, *node.get_kernel_impl_params());
}
bool does_dynamic_implementation_exist(const cldnn::program_node& node, const kernel_impl_params& impl_param) const override {
OPENVINO_ASSERT(node.type() == this, "[GPU] primitive_type_base::does_possible_implementation_exist: primitive type mismatch");
return implementation_map<PType>::check(impl_param, node.get_preferred_impl_type(), shape_types::dynamic_shape);
}
cldnn::layout calc_output_layout(const cldnn::program_node& node, const kernel_impl_params& impl_param) const override {
@@ -82,6 +94,18 @@ struct primitive_type_base : primitive_type {
OPENVINO_ASSERT(node.type() == this, "[GPU] primitive_type_base::to_string: primitive type mismatch");
return typed_primitive_inst<PType>::to_string(node);
}
shape_types get_shape_type(const kernel_impl_params& impl_params) const {
for (auto& in_shape : impl_params.input_layouts) {
if (in_shape.is_dynamic()) {
return shape_types::dynamic_shape;
}
}
if (impl_params.get_output_layout().is_dynamic())
return shape_types::dynamic_shape;
return shape_types::static_shape;
}
};
} // namespace cldnn
src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp
+15 -27
View File
@@ -810,7 +810,17 @@ kernel_selector::tuning_mode to_tuning_mode(cldnn::tuning_mode mode) {
kernel_selector::data_tensor convert_data_tensor(const layout& l, uint32_t split, const tensor view_offset) {
const auto& pad = l.data_padding;
const auto& vals = l.get_tensor().sizes(l.format);
const auto& vals_original = l.get_partial_shape();
// legacy get_tensor().sizes() impl return dims in external order, so we need to transpose dims
ov::PartialShape vals_ordered;
auto axis_order = format::traits(l.format)._order;
for (size_t i = 0; i < axis_order.size(); i++) {
if (axis_order[i] >= vals_original.size())
vals_ordered.push_back(ov::Dimension(1));
else
vals_ordered.push_back(vals_original[axis_order[i]]);
}
const auto& add_offsets = view_offset.sizes(l.format);
const auto& lower_pad = pad.lower_size().sizes(l.format);
const auto& upper_pad = pad.upper_size().sizes(l.format);
@@ -818,46 +828,24 @@ kernel_selector::data_tensor convert_data_tensor(const layout& l, uint32_t split
kernel_selector::n_dims vec(kernel_selector::DataTensor::ChannelsCount(ks_layout));
size_t pitch = 1;
auto new_vals = vals;
if (ks_layout == kernel_selector::Tensor::b_fs_yx_fsv32) {
new_vals[1] = align_to(vals[1], 32);
}
if (ks_layout == kernel_selector::Tensor::bs_fs_yx_bsv16_fsv16) {
new_vals[0] = align_to(vals[0], 16);
new_vals[1] = align_to(vals[1], 16);
}
if (ks_layout == kernel_selector::Tensor::bs_fs_zyx_bsv16_fsv16) {
new_vals[0] = align_to(vals[0], 16);
new_vals[1] = align_to(vals[1], 16);
}
for (size_t i = 0; i < vec.size(); i++) {
const size_t tensor_index = vec.size() - 1 - i;
const auto d = vals[tensor_index];
const auto d = tensor_index < vals_ordered.size() ? vals_ordered[tensor_index] : ov::Dimension(1);
const auto lp = lower_pad[tensor_index] + add_offsets[tensor_index];
const auto up = upper_pad[tensor_index];
// tells us how many elements are reserved in memory for this tensor index
const auto reserved_in_mem_count = new_vals[tensor_index] - add_offsets[tensor_index];
const auto reserved_in_mem_count = d.is_dynamic() ? 0 : d.get_length() - add_offsets[tensor_index];
auto& elm = vec[i];
elm.v = static_cast<size_t>(d - add_offsets[tensor_index]);
elm.v = d.is_dynamic() ? 0 : static_cast<size_t>(d.get_length() - add_offsets[tensor_index]);
elm.pitch = pitch;
elm.pad.before = lp;
elm.pad.after = up;
elm.is_dynamic = d.is_dynamic();
pitch *= (reserved_in_mem_count + lp + up);
}
if (ks_layout == kernel_selector::Tensor::bs_fs_yx_bsv16_fsv16) {
vec[2].pitch = (vec[0].v * vec[1].v) * 16;
vec[3].pitch = vec[2].pitch * vec[2].LogicalDimPadded();
}
if (ks_layout == kernel_selector::Tensor::bs_fs_zyx_bsv16_fsv16) {
vec[3].pitch = (vec[0].v * vec[1].v * vec[2].v) * 16;
vec[4].pitch = vec[3].pitch * vec[3].LogicalDimPadded();
}
const int feature_index =
kernel_selector::DataTensor::Channelndex(ks_layout, kernel_selector::Tensor::DataChannelName::FEATURE);
vec[feature_index].v /= split;
src/plugins/intel_gpu/src/graph/primitive_inst.cpp
+87 -27
View File
@@ -281,41 +281,87 @@ void primitive_inst::realloc_if_needed() {
void primitive_inst::update_impl() {
GPU_DEBUG_PROFILED_STAGE(instrumentation::pipeline_stage::update_implementation);
GPU_DEBUG_GET_INSTANCE(debug_config);
auto prev_impl_str = _impl != nullptr ? _impl->get_kernel_name() : "nullptr";
auto extend_to_6d = [this](ov::PartialShape ps) -> std::vector<size_t> {
// For shape_of we extend shape with 1-s to 6d rank to make kernel simpler
if (_node->is_type<shape_of>()) {
ps.insert(ps.end(), 6 - ps.size(), ov::Dimension(1));
return ps.to_shape();
}
if (ps.size() < 4) {
ps.insert(ps.end(), 4 - ps.size(), ov::Dimension(1));
}
layout l(ps, data_types::i32, format::get_default_format(ps.size()));
return l.transform(format::bfwzyx).to_shape();
};
auto get_layout_key = [&](const kernel_impl_params& params) -> size_t {
size_t seed = 0;
auto& id = params.desc->id;
for (size_t i = 0; i < id.size(); i++) {
seed = hash_combine(seed, id[i]);
}
seed = hash_combine(seed, _node->get_unique_id());
for (auto& layout : params.input_layouts) {
for (auto& d : layout.get_shape()) {
seed = hash_combine(seed, d);
}
}
for (auto& d : params.get_output_layout().get_shape()) {
seed = hash_combine(seed, d);
}
return seed;
};
auto update_shape_info = [this, extend_to_6d, debug_config, prev_impl_str](const kernel_impl_params& params) {
mem_lock<int32_t> lock(_shape_info_memory, _network.get_stream());
size_t offset = 0;
for (size_t i = 0; i < _node->get_dependencies().size(); i++) {
if (_node->get_dependency(i).get_output_layout().is_dynamic()) {
auto input_shape = extend_to_6d(params.get_input_layout(i).get_partial_shape());
for (size_t j = 0; j < input_shape.size(); j++)
lock[offset++] = static_cast<int32_t>(input_shape[j]);
}
}
if (_node->get_output_layout().is_dynamic()) {
auto output_shape = extend_to_6d(params.get_output_layout().get_partial_shape());
for (size_t j = 0; j < output_shape.size(); j++)
lock[offset++] = static_cast<int32_t>(output_shape[j]);
}
GPU_DEBUG_IF(debug_config->verbose >= 4) {
std::stringstream s;
s << "shapes: ";
for (size_t i = 0; i < offset; i++)
s << lock[i] << " ";
GPU_DEBUG_COUT << id() << ": update dynamic impl " << prev_impl_str << " to new shape: " << s.str() << std::endl;
}
};
if (!_node->is_type<data>() && !(_node->is_type<mutable_data>() && _node->get_dependencies().empty())) {
// Update param if fake_alignment is available
auto updated_params = _node->type()->get_fake_aligned_params(*_impl_params);
auto get_layout_key = [&]() -> size_t {
size_t seed = 0;
auto& id = updated_params.desc->id;
for (size_t i = 0; i < id.size(); i++) {
seed = hash_combine(seed, id[i]);
}
seed = hash_combine(seed, _node->get_unique_id());
for (auto& layout : updated_params.input_layouts) {
for (auto& d : layout.get_shape()) {
seed = hash_combine(seed, d);
}
}
for (auto& d : updated_params.get_output_layout().get_shape()) {
seed = hash_combine(seed, d);
}
return seed;
};
auto layout_key = get_layout_key();
auto layout_key = get_layout_key(updated_params);
auto& cache = get_network().get_implementations_cache();
if (cache.has(layout_key)) {
_impl = cache.get(layout_key)->clone();
GPU_DEBUG_PROFILED_STAGE_CACHE_HIT(true);
} else {
_impl = _node->type()->choose_impl(*_node, updated_params);
auto& kernels_cache = get_network().get_kernels_cache();
auto kernel_ids = kernels_cache.add_kernels_source(_impl->get_kernels_source());
_impl->set_kernel_ids(kernel_ids);
kernels_cache.compile();
_impl->init_kernels(kernels_cache);
cache.add(layout_key, _impl->clone());
kernels_cache.reset();
if (_dynamic_impl) {
_impl = _dynamic_impl->clone();
_impl->update_dispatch_data(updated_params);
update_shape_info(updated_params);
} else {
_impl = _node->type()->choose_impl(*_node, updated_params);
auto& kernels_cache = get_network().get_kernels_cache();
auto kernel_ids = kernels_cache.add_kernels_source(_impl->get_kernels_source());
_impl->set_kernel_ids(kernel_ids);
kernels_cache.compile();
_impl->init_kernels(kernels_cache);
cache.add(layout_key, _impl->clone());
kernels_cache.reset();
}
}
reset_shape_change();
@@ -335,6 +381,7 @@ event::ptr primitive_inst::execute(const std::vector<event::ptr>& events) {
std::vector<event::ptr> dependencies;
if (is_dynamic()) {
OPENVINO_ASSERT(_node != nullptr, "[GPU] Invalid primitive_inst object for dynamic shapes case: program_node can't be null");
update_shape();
if (!is_valid_fusion()) {
auto subgraph = get_unfused_subgraph();
@@ -483,6 +530,7 @@ primitive_inst::primitive_inst(network& network)
, _node(nullptr)
, _impl_params(nullptr)
, _impl(nullptr)
, _dynamic_impl(nullptr)
, _outputs({memory::ptr()})
, _output_changed(false)
, _mem_allocated(false) {}
@@ -493,6 +541,7 @@ primitive_inst::primitive_inst(network& network, program_node const& node, bool
, _node_output_layout(node.get_output_layout())
, _impl_params(node.get_kernel_impl_params())
, _impl(node.get_selected_impl() ? node.get_selected_impl()->clone() : nullptr)
, _dynamic_impl(nullptr)
, _outputs({memory::ptr()})
, _output_changed(false)
, _mem_allocated(allocate_memory)
@@ -534,8 +583,19 @@ primitive_inst::primitive_inst(network& network, program_node const& node, bool
_outputs = allocate_outputs();
}
}
if (_impl)
if (_impl) {
_impl->set_node_params(node);
if (_impl->is_dynamic()) {
_dynamic_impl = _impl->clone();
// Actual shape info layout is the following:
// input_0 -> input_1, ..., fused_dep_0, fused_dep1, ..., output_0, output_1, ...
// For each tensor we save 6 dimensions if [bfwzyx] order
const int64_t buffers_count = _node->get_dependencies().size() + _node->get_outputs_count();
const size_t tensor_dims_count = 6;
const int64_t shape_elements = buffers_count * tensor_dims_count;
_shape_info_memory = _network.get_engine().allocate_memory(layout{{shape_elements}, data_types::i32, format::bfyx});
}
}
if (_outputs[0])
max_output_layout_size = _outputs[0]->get_layout().count();
src/plugins/intel_gpu/src/kernel_selector/cl_kernels/permute_ref.cl
+1
View File
@@ -6,6 +6,7 @@
#include "include/batch_headers/fetch_data.cl"
KERNEL (permute_ref)(
OPTIONAL_SHAPE_INFO_ARG
const __global INPUT0_TYPE* input,
__global OUTPUT_TYPE* output
#if HAS_FUSED_OPS_DECLS
src/plugins/intel_gpu/src/kernel_selector/jitter.cpp
+120 -35
View File
@@ -186,6 +186,19 @@ std::string toCodeString(size_t val) {
return buf;
}
std::string toCodeString(const Tensor::Dim& dim, size_t offset, bool padded) {
if (dim.is_dynamic) {
if (padded && dim.pad.Total() > 0) {
snprintf(buf, sizeof(buf), "(shape_info[%zu] + %zu)", offset, dim.pad.Total());
} else {
snprintf(buf, sizeof(buf), "shape_info[%zu]", offset);
}
} else {
snprintf(buf, sizeof(buf), "%zu", dim.v + (padded ? dim.pad.Total() : 0));
}
return buf;
}
std::string toCodeString(float val) {
if (std::isinf(val))
return std::signbit(val) ? "-INFINITY" : "INFINITY";
@@ -259,9 +272,13 @@ public:
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DataTensorJitConstant : public TensorBaseTJitConstant<Datatype, DataLayout> {
const DataTensor _tensor;
const size_t _dyn_array_index;
public:
DataTensorJitConstant(const std::string& name, const DataTensor& t) : TensorBaseTJitConstant(name), _tensor(t) {}
DataTensorJitConstant(const std::string& name, const DataTensor& t, size_t dyn_array_index = 0)
: TensorBaseTJitConstant(name)
, _tensor(t)
, _dyn_array_index(dyn_array_index) {}
JitDefinitions GetDefinitions() const override;
};
@@ -269,32 +286,96 @@ public:
JitDefinitions DataTensorJitConstant::GetDefinitions() const {
JitDefinitions baseDefinitions = TensorBaseTJitConstant::GetDefinitions(_tensor);
JitDefinitions definitions{
{_name + "_SIZE_X", toCodeString(_tensor.X().v)},
{_name + "_SIZE_Y", toCodeString(_tensor.Y().v)},
{_name + "_SIZE_Z", toCodeString(_tensor.Z().v)},
{_name + "_SIZE_W", toCodeString(_tensor.W().v)},
{_name + "_FEATURE_NUM", toCodeString(_tensor.Feature().v)},
{_name + "_BATCH_NUM", toCodeString(_tensor.Batch().v)},
{_name + "_X_PITCH", toCodeString(_tensor.X().pitch)},
{_name + "_Y_PITCH", toCodeString(_tensor.Y().pitch)},
{_name + "_Z_PITCH", toCodeString(_tensor.Z().pitch)},
{_name + "_W_PITCH", toCodeString(_tensor.W().pitch)},
{_name + "_FEATURE_PITCH", toCodeString(_tensor.Feature().pitch)},
{_name + "_BATCH_PITCH", toCodeString(_tensor.Batch().pitch)},
{_name + "_PAD_BEFORE_SIZE_X", toCodeString(_tensor.X().pad.before)},
{_name + "_PAD_BEFORE_SIZE_Y", toCodeString(_tensor.Y().pad.before)},
{_name + "_PAD_BEFORE_SIZE_Z", toCodeString(_tensor.Z().pad.before)},
{_name + "_PAD_BEFORE_SIZE_W", toCodeString(_tensor.W().pad.before)},
{_name + "_PAD_BEFORE_FEATURE_NUM", toCodeString(_tensor.Feature().pad.before)},
{_name + "_PAD_BEFORE_BATCH_NUM", toCodeString(_tensor.Batch().pad.before)},
{_name + "_PAD_AFTER_SIZE_X", toCodeString(_tensor.X().pad.after)},
{_name + "_PAD_AFTER_SIZE_Y", toCodeString(_tensor.Y().pad.after)},
{_name + "_PAD_AFTER_SIZE_Z", toCodeString(_tensor.Z().pad.after)},
{_name + "_PAD_AFTER_SIZE_W", toCodeString(_tensor.W().pad.after)},
{_name + "_PAD_AFTER_FEATURE_NUM", toCodeString(_tensor.Feature().pad.after)},
{_name + "_PAD_AFTER_BATCH_NUM", toCodeString(_tensor.Batch().pad.after)},
};
const size_t idx_offset = _dyn_array_index * 6; // 6D max
JitDefinitions definitions{};
if (_tensor.is_dynamic()) {
auto x = toCodeString(_tensor.X(), idx_offset + 5);
auto y = toCodeString(_tensor.Y(), idx_offset + 4);
auto z = toCodeString(_tensor.Z(), idx_offset + 3);
auto w = toCodeString(_tensor.W(), idx_offset + 2);
auto f = toCodeString(_tensor.Feature(), idx_offset + 1);
auto b = toCodeString(_tensor.Batch(), idx_offset + 0);
auto x_padded = toCodeString(_tensor.X(), idx_offset + 5, true);
auto y_padded = toCodeString(_tensor.Y(), idx_offset + 4, true);
auto z_padded = toCodeString(_tensor.Z(), idx_offset + 3, true);
auto w_padded = toCodeString(_tensor.W(), idx_offset + 2, true);
auto f_padded = toCodeString(_tensor.Feature(), idx_offset + 1, true);
auto b_padded = toCodeString(_tensor.Batch(), idx_offset + 0, true);
auto multiply = [](std::vector<std::string> dims) -> std::string {
std::string res = "(";
for (size_t i = 0; i < dims.size(); i++) {
auto& d = dims[i];
res += d;
if (i != dims.size() - 1)
res += "*";
}
res += ")";
return res;
};
definitions = {
{_name + "_SIZE_X", x},
{_name + "_SIZE_Y", y},
{_name + "_SIZE_Z", z},
{_name + "_SIZE_W", w},
{_name + "_FEATURE_NUM", f},
{_name + "_BATCH_NUM", b},
{_name + "_PAD_BEFORE_SIZE_X", toCodeString(_tensor.X().pad.before)},
{_name + "_PAD_BEFORE_SIZE_Y", toCodeString(_tensor.Y().pad.before)},
{_name + "_PAD_BEFORE_SIZE_Z", toCodeString(_tensor.Z().pad.before)},
{_name + "_PAD_BEFORE_SIZE_W", toCodeString(_tensor.W().pad.before)},
{_name + "_PAD_BEFORE_FEATURE_NUM", toCodeString(_tensor.Feature().pad.before)},
{_name + "_PAD_BEFORE_BATCH_NUM", toCodeString(_tensor.Batch().pad.before)},
{_name + "_PAD_AFTER_SIZE_X", toCodeString(_tensor.X().pad.after)},
{_name + "_PAD_AFTER_SIZE_Y", toCodeString(_tensor.Y().pad.after)},
{_name + "_PAD_AFTER_SIZE_Z", toCodeString(_tensor.Z().pad.after)},
{_name + "_PAD_AFTER_SIZE_W", toCodeString(_tensor.W().pad.after)},
{_name + "_PAD_AFTER_FEATURE_NUM", toCodeString(_tensor.Feature().pad.after)},
{_name + "_PAD_AFTER_BATCH_NUM", toCodeString(_tensor.Batch().pad.after)},
};
if (_tensor.GetLayout() == DataLayout::bf ||
_tensor.GetLayout() == DataLayout::bfyx ||
_tensor.GetLayout() == DataLayout::bfzyx ||
_tensor.GetLayout() == DataLayout::bfwzyx) {
definitions.push_back({_name + "_X_PITCH", "1"});
definitions.push_back({_name + "_Y_PITCH", x_padded});
definitions.push_back({_name + "_Z_PITCH", multiply({x_padded, y_padded})});
definitions.push_back({_name + "_W_PITCH", multiply({x_padded, y_padded, z_padded})});
definitions.push_back({_name + "_FEATURE_PITCH", multiply({x_padded, y_padded, z_padded, w_padded})});
definitions.push_back({_name + "_BATCH_PITCH", multiply({x_padded, y_padded, z_padded, w_padded, f_padded})});
} else {
OPENVINO_ASSERT(false, "[GPU] Jitter couldn't generate dynamic pitches for given layout");
}
} else {
definitions = {
{_name + "_SIZE_X", toCodeString(_tensor.X().v)},
{_name + "_SIZE_Y", toCodeString(_tensor.Y().v)},
{_name + "_SIZE_Z", toCodeString(_tensor.Z().v)},
{_name + "_SIZE_W", toCodeString(_tensor.W().v)},
{_name + "_FEATURE_NUM", toCodeString(_tensor.Feature().v)},
{_name + "_BATCH_NUM", toCodeString(_tensor.Batch().v)},
{_name + "_X_PITCH", toCodeString(_tensor.X().pitch)},
{_name + "_Y_PITCH", toCodeString(_tensor.Y().pitch)},
{_name + "_Z_PITCH", toCodeString(_tensor.Z().pitch)},
{_name + "_W_PITCH", toCodeString(_tensor.W().pitch)},
{_name + "_FEATURE_PITCH", toCodeString(_tensor.Feature().pitch)},
{_name + "_BATCH_PITCH", toCodeString(_tensor.Batch().pitch)},
{_name + "_PAD_BEFORE_SIZE_X", toCodeString(_tensor.X().pad.before)},
{_name + "_PAD_BEFORE_SIZE_Y", toCodeString(_tensor.Y().pad.before)},
{_name + "_PAD_BEFORE_SIZE_Z", toCodeString(_tensor.Z().pad.before)},
{_name + "_PAD_BEFORE_SIZE_W", toCodeString(_tensor.W().pad.before)},
{_name + "_PAD_BEFORE_FEATURE_NUM", toCodeString(_tensor.Feature().pad.before)},
{_name + "_PAD_BEFORE_BATCH_NUM", toCodeString(_tensor.Batch().pad.before)},
{_name + "_PAD_AFTER_SIZE_X", toCodeString(_tensor.X().pad.after)},
{_name + "_PAD_AFTER_SIZE_Y", toCodeString(_tensor.Y().pad.after)},
{_name + "_PAD_AFTER_SIZE_Z", toCodeString(_tensor.Z().pad.after)},
{_name + "_PAD_AFTER_SIZE_W", toCodeString(_tensor.W().pad.after)},
{_name + "_PAD_AFTER_FEATURE_NUM", toCodeString(_tensor.Feature().pad.after)},
{_name + "_PAD_AFTER_BATCH_NUM", toCodeString(_tensor.Batch().pad.after)},
};
}
auto is_common_nd_layout = [](std::vector<Tensor::DataChannelName> common_channels, DataLayout l) -> bool {
for (size_t c = 0; c < static_cast<size_t>(Tensor::DataChannelName::COUNT); c++) {
@@ -471,7 +552,7 @@ JitDefinitions DataTensorJitConstant::GetDefinitions() const {
}
std::string offset = toCodeString(_tensor.GetFirstElementOffset());
if (_tensor.LogicalSize() == 1) {
if (_tensor.LogicalSize() == 1 && !_tensor.is_dynamic()) {
// if tensor contains single element we can always return 0 for safe function
if (_tensor.PitchesDifferFromLogicalDims()) {
definitions.push_back({ safe_index_func_name, offset });
@@ -480,7 +561,7 @@ JitDefinitions DataTensorJitConstant::GetDefinitions() const {
definitions.push_back({ safe_index_func_name, "0" });
definitions.push_back({ index_func_name, "0" });
}
} else if (_tensor.LogicalSize() == _tensor.Feature().v) {
} else if (_tensor.LogicalSize() == _tensor.Feature().v && !_tensor.is_dynamic()) {
// We support broadcast only if corresponding dimension is equal to 1.
// Otherwise, dimensions should be equal and using "f" should be safe.
if (_tensor.PitchesDifferFromLogicalDims() && _tensor.SimpleLayout()) {
@@ -509,8 +590,8 @@ JitDefinitions DataTensorJitConstant::GetDefinitions() const {
return definitions;
}
std::shared_ptr<JitConstant> MakeJitConstant(const std::string& name, const DataTensor& value) {
return std::static_pointer_cast<JitConstant>(std::make_shared<DataTensorJitConstant>(name, value));
std::shared_ptr<JitConstant> MakeJitConstant(const std::string& name, const DataTensor& value, size_t dyn_tensor_index) {
return std::static_pointer_cast<JitConstant>(std::make_shared<DataTensorJitConstant>(name, value, dyn_tensor_index));
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -1494,13 +1575,17 @@ std::string FusedOpsCodeGenerator::GetTypeStr() const {
}
}
JitConstants FusedOpsCodeGenerator::MakeFusedTensorJitConstants(const FusedOpsConfiguration& /*conf*/) const {
JitConstants FusedOpsCodeGenerator::MakeFusedTensorJitConstants(const FusedOpsConfiguration& /*conf*/, size_t dynamic_in_tensors_count) const {
JitConstants jit{};
size_t dyn_tensor_idx = dynamic_in_tensors_count;
for (size_t op_input_id = 0; op_input_id < desc.tensors.size(); op_input_id++) {
std::string name = GetInputTensorName(op_input_id);
jit.AddConstant(MakeJitConstant(name, desc.tensors[op_input_id]));
jit.AddConstant(MakeJitConstant(name, desc.tensors[op_input_id], dyn_tensor_idx));
if (desc.tensors[op_input_id].is_dynamic())
dyn_tensor_idx++;
}
jit.AddConstant(MakeJitConstant(GetOutputTensorName(), desc.output_tensor));
// Use shape_ids from output tensor as won't support fused ops which changes out shape for now
jit.AddConstant(MakeJitConstant(GetOutputTensorName(), desc.output_tensor, dyn_tensor_idx));
return jit;
}
@@ -1526,7 +1611,7 @@ JitConstants FusedOpsCodeGenerator::MakeLoadJitConstants(const FusedOpsConfigura
auto fused_op_config = conf;
std::string load_decls = "";
static int i = 0;
static thread_local int i = 0;
// TODO: check if there is a use case for index reuse or it can be removed
bool reuse_index = false;
bool safe_load = conf.boundary_check == FusedOpsConfiguration::BoundaryCheck::ENABLED;
src/plugins/intel_gpu/src/kernel_selector/jitter.h
+3 -2
View File
@@ -94,6 +94,7 @@ std::string toCodeString(double val);
std::string toCodeString(size_t val);
std::string toCodeString(uint8_t val);
std::string toCodeString(int8_t val);
std::string toCodeString(const Tensor::Dim& dim, size_t offset, bool padded = false);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// JitConstant
@@ -143,7 +144,7 @@ std::shared_ptr<JitConstant> MakeJitConstant(const std::string& name, T value) {
return std::static_pointer_cast<JitConstant>(std::make_shared<simple_jit_constant>(name, toCodeString(value)));
}
std::shared_ptr<JitConstant> MakeJitConstant(const std::string& name, const struct Tensor::DataTensor& value);
std::shared_ptr<JitConstant> MakeJitConstant(const std::string& name, const struct Tensor::DataTensor& value, size_t dyn_tensor_index = 0);
std::shared_ptr<JitConstant> MakeJitConstant(const std::string& name, const struct Tensor::WeightsTensor& value);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -345,7 +346,7 @@ public:
}
};
JitConstants MakeFusedTensorJitConstants(const FusedOpsConfiguration& conf) const;
JitConstants MakeFusedTensorJitConstants(const FusedOpsConfiguration& conf, size_t dynamic_in_out_tensors_count) const;
JitConstants MakeInputDeclsJitConstants(const FusedOpsConfiguration& conf) const;
JitConstants MakeLoadJitConstants(const FusedOpsConfiguration& conf, const DataTensor prim_output) const;
JitConstants MakeOpJitConstants(const FusedOpsConfiguration& conf,
src/plugins/intel_gpu/src/kernel_selector/kernel_base.cpp
+37 -4
View File
@@ -89,14 +89,40 @@ JitConstants KernelBase::MakeBaseParamsJitConstants(const base_params& params) c
jit.Merge(MakeUnitTypeJitConstants(unitType));
jit.Merge(MakeActivationJitConstants(params.activations, unitType));
size_t dyn_tensor_idx = 0;
for (size_t i = 0; i < params.inputs.size(); i++) {
jit.AddConstant(MakeJitConstant("INPUT" + toCodeString(i), params.inputs[i]));
jit.AddConstant(MakeJitConstant("INPUT" + toCodeString(i), params.inputs[i], dyn_tensor_idx));
if (params.inputs[i].is_dynamic())
dyn_tensor_idx++;
}
for (size_t i = 0; i < params.fused_ops.size(); i++) {
auto& fused_op_inputs = params.fused_ops[i].tensors;
for (auto& t : fused_op_inputs) {
if (t.is_dynamic())
dyn_tensor_idx++;
}
}
// NOTE : until all cl kernels legacy is resolved, the outputs are to be OUTPUT, OUTPUT1, OUTPUT2, ...
jit.AddConstant(MakeJitConstant("OUTPUT", params.outputs[0]));
jit.AddConstant(MakeJitConstant("OUTPUT", params.outputs[0], dyn_tensor_idx));
if (params.outputs[0].is_dynamic())
dyn_tensor_idx++;
for (size_t i = 1; i < params.outputs.size(); i++) {
jit.AddConstant(MakeJitConstant("OUTPUT" + toCodeString(i), params.outputs[i]));
jit.AddConstant(MakeJitConstant("OUTPUT" + toCodeString(i), params.outputs[i], dyn_tensor_idx));
if (params.outputs[0].is_dynamic())
dyn_tensor_idx++;
}
if (dyn_tensor_idx > 0) {
jit.AddConstant(MakeJitConstant("IS_DYNAMIC", 1));
jit.AddConstant(MakeJitConstant("OPTIONAL_SHAPE_INFO_ARG", "__global const int* shape_info,"));
jit.AddConstant(MakeJitConstant("OPTIONAL_SHAPE_INFO_TENSOR", "shape_info"));
} else {
jit.AddConstant(MakeJitConstant("OPTIONAL_SHAPE_INFO_ARG", ""));
jit.AddConstant(MakeJitConstant("OPTIONAL_SHAPE_INFO_TENSOR", ""));
}
#ifndef NDEBUG
@@ -187,11 +213,18 @@ JitConstants KernelBase::MakeFusedOpsDeclsJitConstants(const kernel_selector::ba
return jit;
std::string input_decls = "";
size_t dynamic_in_tensors_count = 0;
for (size_t i = 0; i < params.inputs.size(); i++) {
if (params.inputs[i].is_dynamic())
dynamic_in_tensors_count++;
}
for (size_t i = 0; i < params.fused_ops.size(); i++) {
auto fused_dep_codegen = FusedOpsCodeGenerator(params.fused_ops[i]);
std::string op_type = fused_dep_codegen.GetTypeStr();
jit.Merge(fused_dep_codegen.MakeFusedTensorJitConstants(conf[0]));
jit.Merge(fused_dep_codegen.MakeFusedTensorJitConstants(conf[0], dynamic_in_tensors_count));
jit.Merge(fused_dep_codegen.MakeInputDeclsJitConstants(conf[0]));
if (!params.fused_ops[i].tensors.empty()) {
std::string optional_comma = (!input_decls.empty() ? "," : "");
src/plugins/intel_gpu/src/kernel_selector/kernel_base_opencl.cpp
+10 -4
View File
@@ -126,9 +126,13 @@ Arguments KernelBaseOpenCL::GetArgsDesc(uint32_t num_of_input,
bool use_weights,
bool use_bias,
uint32_t number_of_inputs_for_fused_prim,
uint32_t num_of_output) const {
uint32_t num_of_output,
bool is_dynamic) const {
Arguments args;
if (is_dynamic)
args.push_back({ArgumentDescriptor::Types::SHAPE_INFO, 0});
for (uint32_t i = 0; i < num_of_input; i++) {
args.push_back({ArgumentDescriptor::Types::INPUT, i});
}
@@ -198,11 +202,13 @@ void KernelBaseOpenCL::FillCLKernelData(clKernelData& kernel,
bool bias,
int number_of_inputs,
uint32_t number_of_inputs_for_fused_prims,
int number_of_outputs) const {
KernelBase::CheckDispatchData(kernelMapName, dispatchData, engine_info.maxWorkGroupSize);
int number_of_outputs,
bool is_dynamic) const {
if (!is_dynamic)
KernelBase::CheckDispatchData(kernelMapName, dispatchData, engine_info.maxWorkGroupSize);
kernel.code.kernelString = GetKernelString(kernelMapName, jit, entryPoint, engine_info, exeMode);
kernel.params.workGroups.global = dispatchData.gws;
kernel.params.workGroups.local = dispatchData.lws;
kernel.params.arguments = GetArgsDesc(number_of_inputs, weights, bias, number_of_inputs_for_fused_prims, number_of_outputs);
kernel.params.arguments = GetArgsDesc(number_of_inputs, weights, bias, number_of_inputs_for_fused_prims, number_of_outputs, is_dynamic);
}
} // namespace kernel_selector
src/plugins/intel_gpu/src/kernel_selector/kernel_base_opencl.h
+4 -2
View File
@@ -31,7 +31,8 @@ protected:
bool use_weights,
bool use_bias,
uint32_t number_of_inputs_for_fused_prim = 0,
uint32_t num_of_outpus = 1) const;
uint32_t num_of_outpus = 1,
bool is_dynamic = false) const;
std::shared_ptr<KernelString> GetKernelString(const std::string& kernel_name,
const std::pair<std::string, std::string>& jit,
const std::string& entry_point,
@@ -51,6 +52,7 @@ protected:
bool bias = false,
int number_of_inputs = 1,
uint32_t number_of_inputs_for_fused_prims = 0,
int number_of_outputs = 1) const;
int number_of_outputs = 1,
bool is_dynamic = false) const;
};
} // namespace kernel_selector
src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.h
+2
View File
@@ -107,6 +107,8 @@ struct KernelData {
WeightsReorderParams weightsReorderParams;
std::string kernelName;
std::function<void(const Params&, KernelData&)> update_dispatch_data_func = nullptr;
int autoTuneIndex = -1;
bool can_reuse_memory = true;
src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp
+9
View File
@@ -474,6 +474,8 @@ ParamsKey base_params::GetParamsKey() const {
// TODO : multiple output support
bool bFP16Used = (outputs[0].GetDType() == Datatype::F16);
bool dynamic_shapes = false;
for (const auto& i : inputs) {
k.EnableInputDataType(i.GetDType());
k.EnableInputLayout(i.GetLayout());
@@ -483,11 +485,14 @@ ParamsKey base_params::GetParamsKey() const {
bOffests |= (i.GetFirstElementOffset() != 0);
bDifferentTypes |= (i.GetDType() != outputs[0].GetDType());
bFP16Used |= (i.GetDType() == Datatype::F16);
dynamic_shapes |= i.is_dynamic();
}
k.EnableOutputDataType(outputs[0].GetDType());
k.EnableOutputLayout(outputs[0].GetLayout());
dynamic_shapes |= outputs[0].is_dynamic();
if (bBatching) {
k.EnableBatching();
}
@@ -504,6 +509,10 @@ ParamsKey base_params::GetParamsKey() const {
k.EnableTensorOffset();
}
if (dynamic_shapes) {
k.EnableDynamicShapesSupport();
}
if (!engineInfo.bFP16Support && bFP16Used) {
// I'm not sure it's the best idea, but we can live with it right now
k.EnableFP16Emulation();
src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h
+2
View File
@@ -76,6 +76,7 @@ public:
uint32_t sym_quantization : 1;
uint32_t asym_w_quantization : 1;
uint32_t asym_d_quantization : 1;
uint32_t dynamic_shapes : 1;
union dedicated_t {
struct argm_t {
@@ -242,6 +243,7 @@ public:
void EnableFP16Emulation() { key.restrict.val.FP16Emulation = 1; }
void EnableDifferentTypes() { key.restrict.val.different_types = 1; }
void EnableDifferentInputWeightsTypes() { key.restrict.val.different_input_weights_types = 1; }
void EnableDynamicShapesSupport() { key.restrict.val.dynamic_shapes = 1; }
void EnableInputLayout(DataLayout l) { key.inputLayout.set(static_cast<size_t>(l)); }
void EnableAllInputLayout() { key.inputLayout.set(); }
void EnableOutputLayout(DataLayout l) { key.outputLayout.set(static_cast<size_t>(l)); }
src/plugins/intel_gpu/src/kernel_selector/kernels/permute/permute_kernel_base.cpp
+21 -1
View File
@@ -36,10 +36,30 @@ KernelsData PermuteKernelBase::GetKernelsData(const Params& params, const option
auto dispatchData = SetDefault(newParams);
auto cldnn_jit = GetJitConstants(newParams, dispatchData);
kd.update_dispatch_data_func = [this](const Params& params, KernelData& kernel_data) {
const auto& prim_params = static_cast<const permute_params&>(params);
auto dispatchData = SetDefault(prim_params);
OPENVINO_ASSERT(kernel_data.kernels.size() == 1, "[GPU] Invalid kernels size for update dispatch data func");
kernel_data.kernels[0].params.workGroups.global = dispatchData.gws;
kernel_data.kernels[0].params.workGroups.local = dispatchData.lws;
};
auto entry_point = GetEntryPoint(kernelName, newParams.layerID, params, options);
std::pair<std::string, std::string> jit = CreateJit(kernelName, cldnn_jit, entry_point);
auto& kernel = kd.kernels[0];
FillCLKernelData(kernel, dispatchData, params.engineInfo, kernelName, jit, entry_point, "", false, false, 1, GetFusedPrimitiveInputsCount(params));
FillCLKernelData(kernel,
dispatchData,
params.engineInfo,
kernelName,
jit,
entry_point,
"",
false,
false,
1,
GetFusedPrimitiveInputsCount(params),
1,
newParams.outputs[0].is_dynamic());
return {kd};
}
src/plugins/intel_gpu/src/kernel_selector/kernels/permute/permute_kernel_ref.cpp
+1
View File
@@ -118,6 +118,7 @@ ParamsKey PermuteKernelRef::GetSupportedKey() const {
k.EnableTensorOffset();
k.EnableTensorPitches();
k.EnableBatching();
k.EnableDynamicShapesSupport();
return k;
}
src/plugins/intel_gpu/src/kernel_selector/tensor_type.cpp
+1 -1
View File
@@ -338,7 +338,7 @@ NDims DataTensor::GetSimpleDims(const std::vector<size_t>& d, DataLayout l) {
for (size_t i = 0; i < newDims.size(); i++) {
Pad p = {0, newDims[i] - d[i]};
ret[i] = {d[i], pitch, p};
ret[i] = {d[i], pitch, p, d[i] == 0};
pitch *= newDims[i];
}
src/plugins/intel_gpu/src/kernel_selector/tensor_type.h
+5
View File
@@ -240,6 +240,7 @@ struct Dim {
size_t v;
size_t pitch;
Pad pad;
bool is_dynamic;
size_t LogicalDimPadded() const { return v + pad.Total(); }
};
@@ -482,6 +483,10 @@ public:
return differ;
}
bool is_dynamic() const {
return std::any_of(dims.begin(), dims.end(), [](const Dim& d) { return d.is_dynamic; });
}
virtual ~TensorBase() = default;
};
src/plugins/intel_gpu/src/runtime/layout.cpp
+135
View File
@@ -99,6 +99,22 @@ std::pair<bool, bool> are_layouts_identical(layout const& l1, layout const& l2)
return {false, false};
}
std::vector<cldnn::tensor::value_type> convert_dimensions(const std::vector<cldnn::tensor::value_type>& sizes, std::string in_order, std::string out_order) {
std::vector<cldnn::tensor::value_type> new_sizes(out_order.size(), {-1});
for (size_t out_idx = 0; out_idx < out_order.size(); ++out_idx) {
auto channel = out_order[out_idx];
if (channel == '?')
continue;
auto in_idx = in_order.find(channel);
if (in_idx != in_order.npos) {
if (in_idx < sizes.size())
new_sizes[out_idx] = sizes[in_idx];
}
}
return new_sizes;
}
} // namespace
// The definitions below are needed to follow ODR
@@ -477,4 +493,123 @@ bool layout::identical(const layout& other) const {
return are_layouts_identical(*this, other).first;
}
ov::PartialShape layout::transform(cldnn::format new_fmt) const {
if (format == new_fmt) {
return size;
}
cldnn::tensor::value_type default_size = -1;
auto shape = size.to_shape();
std::vector<tensor::value_type> dims(shape.begin(), shape.end());
const cldnn::format default_fmt = cldnn::format::bfwzyx;
auto old_sizes = convert_dimensions(dims, format.order(), default_fmt.internal_order()); // convert to internal order (bfxyzw)
auto val_order = default_fmt.internal_order();
auto new_order = new_fmt.internal_order();
std::vector<tensor::value_type> new_sizes(old_sizes.size(), {default_size});
auto tmp = 1;
auto tmp_z = 1;
auto tmp_w = 1;
for (size_t i = 0; i < default_fmt.order().size(); i++) {
auto c = val_order[i]; //bfxywz
// skip f, y, z, and w for the formats that do not have it
if (((new_fmt == format::bs_xs_xsv8_bsv8) ||
(new_fmt == format::bs_xs_xsv8_bsv16) ||
(new_fmt == format::os_i_osv8__ai8) ||
(new_fmt == format::os_i_osv16__ai8) ||
(new_fmt == format::bs_x_bsv16)) &&
((c == 'f') ||
(c == 'y') ||
(c == 'z') ||
(c == 'w'))) {
if (new_order[i] == '?')
new_sizes[i] = default_size;
tmp *= old_sizes[i]; //0f0ywz
continue;
}
// skip z for the formats that do not have it
if (((new_fmt != format::bfzyx && new_fmt != format::b_fs_zyx_fsv16 && new_fmt != format::b_fs_zyx_fsv32 &&
new_fmt != format::bfwzyx && new_fmt != format::bs_fs_zyx_bsv16_fsv16 && new_fmt != format::bs_fs_zyx_bsv16_fsv32 &&
new_fmt != format::bs_fs_zyx_bsv32_fsv16 && new_fmt != format::bs_fs_zyx_bsv32_fsv32 &&
new_fmt != format::b_fs_zyx_fsv2 && new_fmt != format::b_fs_zyx_fsv4 &&
new_fmt != format::bs_fs_zyx_bsv8_fsv2 && new_fmt != format::bs_fs_zyx_bsv8_fsv4)) && (c == 'z')) {
if (new_order[i] == '?')
new_sizes[i] = default_size;
tmp_z *= old_sizes[i]; //00000z
continue;
}
if (new_fmt != format::bfwzyx && c == 'w') {
if (new_order[i] == '?')
new_sizes[i] = default_size;
if (new_fmt == format::bfzyx || new_fmt == format::b_fs_zyx_fsv16 ||
new_fmt == format::bs_fs_zyx_bsv16_fsv16 || new_fmt == format::b_fs_zyx_fsv32 ||
new_fmt == format::bs_fs_zyx_bsv16_fsv32)
tmp_w *= old_sizes[i]; //0000w0
else
tmp_z *= old_sizes[i]; //0000w0
continue;
}
auto new_pos = new_order.find(c);
if (new_pos == std::string::npos)
throw std::invalid_argument("cannot convert to new format");
new_sizes[new_pos] = old_sizes[i];
}
// in case of formats with smaller number of dimensions than input, flatten is performed below
if (tmp != 1 || tmp_z != 1 || tmp_w != 1) {
for (size_t i = 0; i < default_fmt.order().size(); i++) {
auto c = val_order[i];
if (c == 'x') {
auto new_pos = new_order.find(c);
new_sizes[new_pos] *= tmp;
}
if (c == 'y') {
auto new_pos = new_order.find(c);
if (new_pos != std::string::npos)
new_sizes[new_pos] *= tmp_z;
}
if (c == 'z') {
auto new_pos = new_order.find(c);
if (new_pos != std::string::npos)
new_sizes[new_pos] *= tmp_w;
}
}
}
for (size_t i = 0; i < new_order.size(); i++) {
auto c = new_order[i]; //bfxywz
if (c == '?')
continue;
if (new_sizes[i] == -1) {
new_sizes[i] = 1;
}
}
auto new_dims = convert_dimensions(new_sizes, default_fmt.internal_order(), new_fmt.order());
for (int idx = (new_dims.size() - 1); idx >= 0; idx--) {
if (new_dims[idx] == -1)
new_dims.erase((new_dims.begin() + idx));
else if (new_dims[idx] < 0)
new_dims[idx] *= -1;
}
ov::Shape new_shape(new_dims.begin(), new_dims.end());
return ov::PartialShape(new_shape);
}
} // namespace cldnn
src/plugins/intel_gpu/src/runtime/ocl/ocl_stream.cpp
+13
View File
@@ -247,6 +247,19 @@ void set_arguments_impl(ocl_kernel_type& kernel,
status = kernel.setArg(i, dynamic_cast<const ocl::gpu_buffer&>(*data.cell).get_buffer());
}
break;
case args_t::SHAPE_INFO:
if (args[i].index == 0 && data.shape_info) {
const auto& shape_info_mem = data.shape_info;
if (shape_info_mem) {
if (shape_info_mem->get_layout().format.is_image_2d())
status = kernel.setArg(i, std::dynamic_pointer_cast<const ocl::gpu_image2d>(shape_info_mem)->get_buffer());
else if (memory_capabilities::is_usm_type(shape_info_mem->get_allocation_type()))
status = kernel.setArgUsm(i, std::dynamic_pointer_cast<const ocl::gpu_usm>(shape_info_mem)->get_buffer());
else
status = kernel.setArg(i, std::dynamic_pointer_cast<const ocl::gpu_buffer>(shape_info_mem)->get_buffer());
}
}
break;
default:
break;
}
src/plugins/intel_gpu/tests/test_cases/permute_gpu_test.cpp
+58
View File
@@ -14,6 +14,8 @@
#include <intel_gpu/primitives/reshape.hpp>
#include <intel_gpu/primitives/crop.hpp>
#include "permute_inst.h"
#include <cmath>
#include <limits>
#include <type_traits>
@@ -1844,3 +1846,59 @@ TEST_P(permute_tile_fsv_5d, i64) {
auto p = GetParam();
run_test<cldnn::data_types::i64>(p.sizes, p.format_fsv);
}
TEST(permute_gpu_f32_dynamic, bfyx_0_2_3_1) {
constexpr size_t array_size = 100;
auto& engine = get_test_engine();
auto input_layout_dynamic = layout{ov::PartialShape::dynamic(4), data_types::f32, format::bfyx};
auto input_layout_static = layout{ov::PartialShape{2, 5, 5, 2}, data_types::f32, format::bfyx};
auto input = engine.allocate_memory(input_layout_static);
std::vector<float> input_data;
input_data.reserve(array_size);
for (size_t i = 0; i < array_size; ++i)
input_data.push_back(static_cast<float>(i));
set_values(input, input_data);
topology topology(
input_layout("input", input_layout_dynamic),
permute("permute", "input", { 0, 2, 3, 1 }));
build_options bo;
bo.set_option(build_option::allow_new_shape_infer(true));
network network(engine, topology, bo);
network.set_input_data("input", input);
auto inst = network.get_primitive("permute");
auto impl = inst->get_impl();
ASSERT_TRUE(impl != nullptr);
ASSERT_TRUE(impl->is_dynamic());
auto outputs = network.execute();
EXPECT_EQ(outputs.size(), size_t(1));
EXPECT_EQ(outputs.begin()->first, "permute");
auto output = outputs.begin()->second.get_memory();
float answers[array_size] = {
0.f, 10.f, 20.f, 30.f, 40.f, 1.f, 11.f, 21.f, 31.f, 41.f,
2.f, 12.f, 22.f, 32.f, 42.f, 3.f, 13.f, 23.f, 33.f, 43.f,
4.f, 14.f, 24.f, 34.f, 44.f, 5.f, 15.f, 25.f, 35.f, 45.f,
6.f, 16.f, 26.f, 36.f, 46.f, 7.f, 17.f, 27.f, 37.f, 47.f,
8.f, 18.f, 28.f, 38.f, 48.f, 9.f, 19.f, 29.f, 39.f, 49.f,
50.f, 60.f, 70.f, 80.f, 90.f, 51.f, 61.f, 71.f, 81.f, 91.f,
52.f, 62.f, 72.f, 82.f, 92.f, 53.f, 63.f, 73.f, 83.f, 93.f,
54.f, 64.f, 74.f, 84.f, 94.f, 55.f, 65.f, 75.f, 85.f, 95.f,
56.f, 66.f, 76.f, 86.f, 96.f, 57.f, 67.f, 77.f, 87.f, 97.f,
58.f, 68.f, 78.f, 88.f, 98.f, 59.f, 69.f, 79.f, 89.f, 99.f
};
cldnn::mem_lock<float> output_ptr(output, get_test_stream());
for (size_t i = 0; i < array_size; i++) {
EXPECT_FLOAT_EQ(answers[i], output_ptr[i]);
}
}