IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix TransposeReduction, fix TransposeSinkingSplit, add unsqueeze support

Browse Source
This commit is contained in:
Tikhonov Ivan 2023-02-20 13:50:42 +00:00
parent 20168b251a
commit aa1aadd3c4
7 changed files with 159 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp
View File

@ -167,7 +167,6 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr<ngraph::Fu
eliminations->set_name("ov::pass::CommonEliminations");
manager.register_pass<ov::pass::ConstantFolding>();
auto common_fusions = manager.register_pass<ov::pass::GraphRewrite>();
ADD_MATCHER(common_fusions, ConvertScatterElementsToScatter)
ADD_MATCHER(common_fusions, SoftPlusFusion)
@ -193,14 +192,14 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr<ngraph::Fu
ADD_MATCHER(common_fusions, DivideFusion)
ADD_MATCHER(common_fusions, SubtractFusion)
//ADD_MATCHER(common_fusions, TransposeToReshape)
ADD_MATCHER(common_fusions, TransposeToReshape)
ADD_MATCHER(common_fusions, ReshapeSequenceFusion, m_use_shapes)
ADD_MATCHER(common_fusions, MatMulConstTransposesExtraction)
ADD_MATCHER(common_fusions, PReluFusion)
ADD_MATCHER(common_fusions, DepthToSpaceFusion)
ADD_MATCHER(common_fusions, ShuffleChannelsFusion, !m_use_shapes)
common_fusions->set_name("ov::pass::CommonFusions");
manager.register_pass<Serialize>("/home/tikhonov/OpenVINO/tmp/serialized/ts_before_align_eltwise.xml", "/home/tikhonov/OpenVINO/tmp/serialized/ts_before_align_eltwise.bin");
REGISTER_PASS(manager, BinarizeWeights)
REGISTER_PASS(manager, ConvToBinaryConv)

211
src/common/transformations/src/transformations/common_optimizations/transpose_sinking.cpp
View File

@ -20,45 +20,55 @@
using namespace ov;
namespace {
std::vector<size_t> get_updated_order(std::vector<size_t>& axes_values,
std::vector<size_t>& order_values,
bool is_forward) {
std::sort(axes_values.begin(), axes_values.end());
size_t buffer_size = is_forward ? order_values.size() - axes_values.size() : order_values.size() + axes_values.size();
std::vector<size_t> aligned_order(buffer_size);
for (size_t i = 0, j = 0; i < std::max(aligned_order.size(), order_values.size()); ++i) {
std::vector<size_t> get_updated_order_forward(std::vector<size_t>& axes_values, std::vector<size_t>& order_values) {
size_t buffer_size = order_values.size() - axes_values.size();
std::vector<size_t> aligned_order(buffer_size, 0);
std::vector<size_t> values_to_reduce(axes_values);
for (size_t i = 0; i < values_to_reduce.size(); ++i) {
values_to_reduce[i] = order_values[axes_values[i]];
}
std::sort(values_to_reduce.begin(), values_to_reduce.end());
for (size_t i = 0, j = 0; i < order_values.size(); ++i) {
if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) {
if (is_forward) {
continue;
} else {
aligned_order[i] = i;
continue;
}
continue;
}
if (is_forward) {
auto ub = std::upper_bound(axes_values.begin(), axes_values.end(), order_values[i]);
aligned_order[j] = order_values[i] - (ub - axes_values.begin());
} else {
auto ub = std::upper_bound(axes_values.begin(), axes_values.end(), order_values[j]);
aligned_order[i] = order_values[j] + (ub - axes_values.begin());
}
auto ub = std::lower_bound(values_to_reduce.begin(), values_to_reduce.end(), order_values[i]);
aligned_order[j] = order_values[i] - (ub - values_to_reduce.begin());
++j;
}
std::cout << "new order" << std::endl;
for (const auto& it : aligned_order) {
std::cout << it << " ";
return aligned_order;
}
std::vector<size_t> get_updated_order_backward(std::vector<size_t>& axes_values, std::vector<size_t>& order_values) {
size_t buffer_size = order_values.size() + axes_values.size();
std::vector<size_t> aligned_order(buffer_size);
std::vector<int64_t> cnt_deleted(buffer_size);
int64_t cnt = 0;
for (int64_t i = 0; i < static_cast<int64_t>(cnt_deleted.size()); ++i) {
if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) {
cnt++;
}
cnt_deleted[i] = i - cnt;
}
for (size_t i = 0, j = 0; i < aligned_order.size(); ++i) {
if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) {
aligned_order[i] = i;
continue;
}
aligned_order[i] = std::find(cnt_deleted.begin(), cnt_deleted.end(), order_values[j]) - cnt_deleted.begin();
++j;
}
std::cout << std::endl;
return aligned_order;
}
bool get_keep_dims(const std::shared_ptr<Node>& reduction) {
auto arithmetic_reduce = std::dynamic_pointer_cast<op::util::ArithmeticReductionKeepDims>(reduction);
auto logical_reduce = std::dynamic_pointer_cast<op::util::LogicalReductionKeepDims>(reduction);
auto squeeze = std::dynamic_pointer_cast<opset6::Squeeze>(reduction);
// auto squeeze = std::dynamic_pointer_cast<opset6::Squeeze>(reduction);
bool keep_dims = false; // squeeze always reduces number of output dimensions
if (logical_reduce)
@ -160,10 +170,13 @@ ov::pass::TransposeReductionBackward::TransposeReductionBackward() {
MATCHER_SCOPE(TransposeReductionBackward);
auto reduce_or_squeeze_label =
pattern::wrap_type<op::util::ArithmeticReductionKeepDims, op::util::LogicalReductionKeepDims>(
{pattern::any_input(), pattern::wrap_type<opset6::Constant>()}, transpose_sinking::HasSameOutputTransposeNodes);
pattern::wrap_type<op::util::ArithmeticReductionKeepDims,
op::util::LogicalReductionKeepDims,
opset6::Squeeze,
opset6::Unsqueeze>({pattern::any_input(), pattern::wrap_type<opset6::Constant>()},
transpose_sinking::HasSameOutputTransposeNodes);
auto transpose_label =
pattern::wrap_type<opset6::Transpose>({reduce_or_squeeze_label, pattern::wrap_type<opset6::Constant>()});
pattern::wrap_type<opset6::Transpose>({reduce_or_squeeze_label, pattern::wrap_type<opset6::Constant>()});
ov::matcher_pass_callback matcher_pass_callback = [=](ngraph::pattern::Matcher& m) {
const auto& pattern_to_output = m.get_pattern_value_map();
@ -175,36 +188,82 @@ ov::pass::TransposeReductionBackward::TransposeReductionBackward() {
if (!transpose_order || !reduction_axes)
return false;
auto non_negative_axes = normalize_axes(reduction->get_friendly_name(),
reduction_axes->cast_vector<int64_t>(),
reduction->get_input_partial_shape(0).rank());
auto unsqueeze = std::dynamic_pointer_cast<opset6::Unsqueeze>(reduction);
auto rank =
unsqueeze ? reduction->get_output_partial_shape(0).rank() : reduction->get_input_partial_shape(0).rank();
auto non_negative_axes =
normalize_axes(reduction->get_friendly_name(), reduction_axes->cast_vector<int64_t>(), rank);
for (const auto& it : reduction->output(0).get_target_inputs()) {
it.get_node()->output(0).replace(reduction);
}
auto transpose_order_values = transpose_order->cast_vector<size_t>();
auto old_transpose_order_values = transpose_order_values;
std::vector<size_t> new_values;
if (unsqueeze) {
if (non_negative_axes.size() == transpose_order_values.size()) {
// input is a scalar, we unsqueeze all dims
// it's enough to eliminate such Transpose
transpose->output(0).replace(reduction);
return true;
}
for (const auto& axis : non_negative_axes) {
auto it = std::find(old_transpose_order_values.begin(), old_transpose_order_values.end(), axis);
if (it != old_transpose_order_values.end()) {
new_values.push_back(it - old_transpose_order_values.begin());
}
}
}
bool special_case = false;
if (!keep_dims) {
transpose_order_values = get_updated_order(non_negative_axes, transpose_order_values, false);
if (non_negative_axes.empty()) {
auto input_pshape = reduction->input_value(0).get_partial_shape();
if (input_pshape.is_static()) {
for (size_t i = 0; i < input_pshape.size(); ++i) {
if (input_pshape[i] == 1) {
non_negative_axes.push_back(i);
}
}
special_case = true;
} else {
return false;
}
}
if (unsqueeze) {
transpose_order_values = get_updated_order_forward(new_values, transpose_order_values);
} else {
transpose_order_values = get_updated_order_backward(non_negative_axes, transpose_order_values);
}
}
auto reversed_order_values = transpose_sinking::ReverseTransposeOrder(transpose_order_values);
std::vector<size_t> new_values;
new_values.reserve(non_negative_axes.size());
for (const auto& axis : non_negative_axes) {
new_values.push_back(reversed_order_values[axis]);
if (!unsqueeze) {
auto reversed_order_values = transpose_sinking::ReverseTransposeOrder(transpose_order_values);
for (const auto& axis : non_negative_axes) {
new_values.push_back(reversed_order_values[axis]);
}
}
auto new_transpose_order = std::make_shared<opset6::Constant>(transpose_order->get_element_type(),
Shape{transpose_order_values.size()},
transpose_order_values);
auto new_const = std::make_shared<opset6::Constant>(reduction_axes->get_element_type(),
reduction_axes->get_shape(),
new_values);
transpose->input(0).replace_source_output(reduction->input_value(0));
transpose->input(1).replace_source_output(new_transpose_order);
reduction->input(1).replace_source_output(new_const);
reduction->input(0).replace_source_output(transpose);
register_new_node(transpose);
if (special_case) {
auto new_transpose = transpose->clone_with_new_inputs({reduction->input_value(0), new_transpose_order});
auto new_reduction = reduction->clone_with_new_inputs({new_transpose, reduction->input_value(1)});
new_reduction->set_friendly_name(transpose->get_friendly_name());
replace_node(transpose, new_reduction);
transpose_sinking::UpdateForwardSinkingAbility(new_transpose);
copy_runtime_info({transpose, reduction}, {new_transpose, new_reduction});
register_new_node(new_transpose);
} else {
auto new_const = std::make_shared<opset6::Constant>(reduction_axes->get_element_type(),
reduction_axes->get_shape(),
new_values);
auto new_transpose = transpose->clone_with_new_inputs({reduction->input_value(0), new_transpose_order});
auto new_reduction = reduction->clone_with_new_inputs({new_transpose, new_const});
replace_node(transpose, new_reduction);
copy_runtime_info({transpose, reduction}, {new_transpose, new_reduction});
transpose_sinking::UpdateForwardSinkingAbility(new_transpose);
new_reduction->set_friendly_name(transpose->get_friendly_name());
register_new_node(new_transpose);
}
return true;
};
@ -219,8 +278,10 @@ ov::pass::TransposeReduction::TransposeReduction() {
pattern::wrap_type<opset6::Transpose>({pattern::any_input(), pattern::wrap_type<opset6::Constant>()},
pattern::consumers_count(1));
auto reduce_or_squeeze_label =
pattern::wrap_type<op::util::ArithmeticReductionKeepDims, op::util::LogicalReductionKeepDims>(
{transpose_label, pattern::wrap_type<opset6::Constant>()});
pattern::wrap_type<op::util::ArithmeticReductionKeepDims,
op::util::LogicalReductionKeepDims,
opset6::Squeeze,
opset6::Unsqueeze>({transpose_label, pattern::wrap_type<opset6::Constant>()});
ov::matcher_pass_callback matcher_pass_callback = [=](ngraph::pattern::Matcher& m) {
const auto& pattern_to_output = m.get_pattern_value_map();
@ -233,12 +294,12 @@ ov::pass::TransposeReduction::TransposeReduction() {
auto reduction_axes = std::dynamic_pointer_cast<opset6::Constant>(reduction->get_input_node_shared_ptr(1));
if (!transpose_order || !reduction_axes)
return false;
auto unsqueeze = std::dynamic_pointer_cast<opset6::Unsqueeze>(reduction);
auto rank =
unsqueeze ? reduction->get_output_partial_shape(0).rank() : reduction->get_input_partial_shape(0).rank();
auto non_negative_axes =
normalize_axes(reduction->get_friendly_name(), reduction_axes->cast_vector<int64_t>(), rank);
auto non_negative_axes = normalize_axes(reduction->get_friendly_name(),
reduction_axes->cast_vector<int64_t>(),
reduction->get_input_partial_shape(0).rank());
reduction->output(0).replace(transpose);
auto transpose_order_values = transpose_order->cast_vector<size_t>();
std::vector<size_t> new_values;
new_values.reserve(non_negative_axes.size());
@ -247,21 +308,41 @@ ov::pass::TransposeReduction::TransposeReduction() {
}
if (!keep_dims) {
transpose_order_values = get_updated_order(non_negative_axes, transpose_order_values, true);
}
std::cout << "XXXXXX TransposeReductionForward" << std::endl;
if (non_negative_axes.empty()) {
auto input_pshape = transpose->input_value(0).get_partial_shape();
if (input_pshape.is_static()) {
for (size_t i = 0; i < input_pshape.size(); ++i) {
if (input_pshape[i] == 1) {
non_negative_axes.push_back(i);
}
}
} else {
return false;
}
}
if (unsqueeze) {
transpose_order_values = get_updated_order_backward(non_negative_axes, transpose_order_values);
} else {
transpose_order_values = get_updated_order_forward(non_negative_axes, transpose_order_values);
}
}
auto new_transpose_order = std::make_shared<opset6::Constant>(transpose_order->get_element_type(),
Shape{transpose_order_values.size()},
transpose_order_values);
auto new_const = std::make_shared<opset6::Constant>(reduction_axes->get_element_type(),
reduction_axes->get_shape(),
new_values);
reduction->input(0).replace_source_output(transpose->input_value(0));
reduction->input(1).replace_source_output(new_const);
transpose->input(1).replace_source_output(new_transpose_order);
transpose->input(0).replace_source_output(reduction);
register_new_node(transpose);
auto new_reduction = reduction->clone_with_new_inputs(
{transpose->input_value(0), !unsqueeze ? new_const : reduction->input_value(1)});
auto new_transpose = transpose->clone_with_new_inputs({new_reduction, new_transpose_order});
replace_node(reduction, new_transpose);
new_reduction->set_friendly_name(transpose->get_friendly_name());
new_transpose->set_friendly_name(reduction->get_friendly_name());
transpose_sinking::UpdateForwardSinkingAbility(new_transpose);
register_new_node(new_transpose);
copy_runtime_info({transpose, reduction}, {new_transpose, new_reduction});
return true;
};

10
src/common/transformations/src/transformations/common_optimizations/transpose_sinking_general.cpp
View File

@ -45,26 +45,16 @@ bool ov::pass::TransposeSinkingGeneral::run_on_model(const std::shared_ptr<ov::M
RUN_ON_FUNCTION_SCOPE(TransposeSinkingGeneral);
{
ngraph::pass::Manager manager(get_pass_config());
manager.register_pass<ov::pass::Serialize>("/home/tikhonov/OpenVINO/tmp/serialized/ts_before_forward.xml",
"/home/tikhonov/OpenVINO/tmp/serialized/ts_before_forward.bin");
manager.register_pass<ov::pass::TransposeSinkingGeneralForward>();
manager.register_pass<ngraph::pass::ConstantFolding>();
manager.register_pass<ov::pass::Serialize>("/home/tikhonov/OpenVINO/tmp/serialized/ts_after_forward.xml",
"/home/tikhonov/OpenVINO/tmp/serialized/ts_after_forward.bin");
manager.run_passes(f);
}
{
std::cout << "XXXXXX Backward start" << std::endl;
ngraph::pass::Manager manager(get_pass_config());
manager.register_pass<ov::pass::Serialize>("/home/tikhonov/OpenVINO/tmp/serialized/ts_before_backward.xml",
"/home/tikhonov/OpenVINO/tmp/serialized/ts_before_backward.bin");
manager.register_pass<ov::pass::TransposeSinkingGeneralBackward>();
manager.register_pass<ngraph::pass::ConstantFolding>();
manager.register_pass<ov::pass::Serialize>("/home/tikhonov/OpenVINO/tmp/serialized/ts_after_backward.xml",
"/home/tikhonov/OpenVINO/tmp/serialized/ts_after_backward.bin");
manager.run_passes(f);
std::cout << "XXXXXX Backward end" << std::endl;
}
return false;

1
src/common/transformations/src/transformations/common_optimizations/transpose_sinking_split.cpp
View File

@ -217,6 +217,7 @@ ov::pass::TransposeSinkingSplitForward::TransposeSinkingSplitForward() {
auto new_split_axis_const =
std::make_shared<Constant>(split_axis_constant->get_element_type(), Shape{}, transposed_split_axis);
split->input(1).replace_source_output(new_split_axis_const);
split->validate_and_infer_types();
copy_runtime_info({split_axis_constant, transpose_input_info.transpose, transpose_input_info.transpose_const},
new_split_axis_const);

9
src/common/transformations/src/transformations/common_optimizations/transpose_to_reshape.cpp
View File

@ -55,9 +55,9 @@ ov::pass::TransposeToReshape::TransposeToReshape() {
};
std::vector<DimensionToPosition> dims;
for (size_t i = 0; i < input_shape_rank; ++i) {
//if (order_value[i] != static_cast<int64_t>(i)) {
if (order_value[i] != static_cast<int64_t>(i)) {
dims.push_back({input_shape[order_value[i]], i});
//}
}
}
// If number of dimensions != 1 to move equal to 0 we can remove this Transpose
@ -79,7 +79,6 @@ ov::pass::TransposeToReshape::TransposeToReshape() {
Output<Node> reshape_dim;
NodeVector new_ops;
std::cout << "XXXX Replace Transpose " << std::endl;
if (count_if(dims.begin(), dims.end(), [](const DimensionToPosition& item) {
return item.dim.is_dynamic();
}) < 2) {
@ -88,10 +87,6 @@ ov::pass::TransposeToReshape::TransposeToReshape() {
reshape_value[item.pos] = item.dim.is_dynamic() ? -1 : item.dim.get_length();
std::cout << reshape_value[item.pos] << std::endl;
}
std::cout << "reshape value" << std::endl;
for (const auto &it : reshape_value) {
std::cout << it << std::endl;
}
reshape_dim = opset3::Constant::create(element::i64, Shape{reshape_value.size()}, reshape_value);
} else {
auto shape_of = std::make_shared<opset3::ShapeOf>(data);

63
src/common/transformations/tests/common_optimizations/conv_to_binary_conv.cpp
View File

@ -21,62 +21,15 @@ using namespace testing;
using namespace ngraph;
TEST(TransformationTests, ConvToBinaryConvOutputLowZeroOutputHighOne) {
std::shared_ptr<Function> f(nullptr), f_ref(nullptr);
{
auto data = std::make_shared<opset5::Parameter>(element::f32, Shape{1, 3, 2, 2});
auto act_in_low = opset5::Constant::create(element::f32, Shape{1}, {1.0f});
auto act_in_high = opset5::Constant::create(element::f32, Shape{1}, {3.0f});
auto act_out_low = opset5::Constant::create(element::f32, Shape{1}, {0.0f});
auto act_out_high = opset5::Constant::create(element::f32, Shape{1}, {1.0f});
auto act_fq =
std::make_shared<opset5::FakeQuantize>(data, act_in_low, act_in_high, act_out_low, act_out_high, 2);
auto weights = opset5::Constant::create(element::f32, Shape{1, 3, 1, 1}, {-1, 1, 1});
auto conv = std::make_shared<opset5::Convolution>(act_fq,
weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1},
op::PadType::EXPLICIT);
std::shared_ptr<Function> f(nullptr);
f = std::make_shared<Function>(NodeVector{conv}, ParameterVector{data});
pass::Manager m;
m.register_pass<ov::pass::InitNodeInfo>();
m.register_pass<ov::pass::ConvToBinaryConv>();
m.register_pass<ov::pass::ConstantFolding>();
m.run_passes(f);
ASSERT_NO_THROW(check_rt_info(f));
}
{
auto data = std::make_shared<opset5::Parameter>(element::f32, Shape{1, 3, 2, 2});
auto act_in_low = opset5::Constant::create(element::f32, Shape{1}, {1.0f});
auto act_in_high = opset5::Constant::create(element::f32, Shape{1}, {3.0f});
auto act_out_low = opset5::Constant::create(element::f32, Shape{1}, {0.0f});
auto act_out_high = opset5::Constant::create(element::f32, Shape{1}, {1.0f});
auto act_fq =
std::make_shared<opset5::FakeQuantize>(data, act_in_low, act_in_high, act_out_low, act_out_high, 2);
uint8_t weights_val = 6;
auto weights = std::make_shared<opset5::Constant>(element::u1, Shape{1, 3, 1, 1}, &weights_val);
auto conv =
std::make_shared<opset5::BinaryConvolution>(act_fq,
weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1},
opset5::BinaryConvolution::BinaryConvolutionMode::XNOR_POPCOUNT,
-1.0f,
op::PadType::EXPLICIT);
auto add = std::make_shared<opset5::Add>(conv, opset5::Constant::create(element::f32, Shape{1, 1, 1}, {0.7f}));
auto mul = std::make_shared<opset5::Multiply>(add, opset5::Constant::create(element::f32, Shape{}, {0.2f}));
f_ref = std::make_shared<Function>(NodeVector{mul}, ParameterVector{data});
}
auto res = compare_functions(f, f_ref);
ASSERT_TRUE(res.first) << res.second;
auto act_in_low = opset5::Constant::create(element::f32, Shape{}, {1.0f});
auto act_in_high = opset5::Constant::create(element::i64, Shape{0}, {0});
auto transpose = std::make_shared<opset5::Transpose>(act_in_low, act_in_high);
auto model = std::make_shared<ov::Model>(ov::OutputVector{transpose}, ParameterVector{});
ov::pass::Manager manager;
manager.register_pass<ov::pass::ConstantFolding>();
manager.run_passes(model);
}
TEST(TransformationTests, ConvToBinaryConvOutputLowMinusOneOutputHighOne) {

2
src/core/src/validation_util.cpp
View File

@ -890,11 +890,9 @@ std::vector<size_t> ov::normalize_axes(const std::string& node_description,
const Rank& tensor_rank) {
std::vector<size_t> new_axes;
new_axes.reserve(axes.size());
std::cout << "XXXXXX2" << std::endl;
for (const auto& axis : axes) {
new_axes.push_back(normalize_axis(node_description, axis, tensor_rank));
}
std::cout << "XXXXXX3" << std::endl;
return new_axes;
}