From 0c041d7ebcc0fbcc68c3ebbf7f1136d3fad229c4 Mon Sep 17 00:00:00 2001 From: Jan Iwaszkiewicz Date: Mon, 20 Nov 2023 15:29:46 +0100 Subject: [PATCH] [PyOV] Adjust opset13 Constant to utilize runtime.op.Constant functionalities (#21046) --- .../python/src/openvino/helpers/packing.py | 4 +- .../src/openvino/runtime/op/__init__.py | 9 - .../src/openvino/runtime/opset13/__init__.py | 2 +- .../src/openvino/runtime/opset13/ops.py | 83 +++- .../src/pyopenvino/graph/ops/constant.cpp | 35 +- .../python/tests/test_graph/test_constant.py | 358 ++++++++++++++++++ 6 files changed, 472 insertions(+), 19 deletions(-) create mode 100644 src/bindings/python/tests/test_graph/test_constant.py diff --git a/src/bindings/python/src/openvino/helpers/packing.py b/src/bindings/python/src/openvino/helpers/packing.py index 2b94c7ab116..523c356ce5b 100644 --- a/src/bindings/python/src/openvino/helpers/packing.py +++ b/src/bindings/python/src/openvino/helpers/packing.py @@ -20,7 +20,7 @@ def pack_data(array: np.ndarray, type: Type) -> np.ndarray: :param array: numpy array with values to pack. :type array: numpy array - :param type: Type to interpret the array values. Type must be u1, u4 or i4. + :param type: Type to interpret the array values. Type must be u1, u4, i4 or nf4. :type type: openvino.runtime.Type """ assert type in [Type.u1, Type.u4, Type.i4, Type.nf4], "Packing algorithm for the" "data types stored in 1, 2 or 4 bits" @@ -57,7 +57,7 @@ def unpack_data(array: np.ndarray, type: Type, shape: Union[list, Shape]) -> np. :param array: numpy array to unpack. :type array: numpy array - :param type: Type to extract from array values. Type must be u1, u4 or i4. + :param type: Type to extract from array values. Type must be u1, u4, i4 or nf4. :type type: openvino.runtime.Type :param shape: the new shape for the unpacked array. :type shape: Union[list, openvino.runtime.Shape] diff --git a/src/bindings/python/src/openvino/runtime/op/__init__.py b/src/bindings/python/src/openvino/runtime/op/__init__.py index 2d740e87cf1..069976998f4 100644 --- a/src/bindings/python/src/openvino/runtime/op/__init__.py +++ b/src/bindings/python/src/openvino/runtime/op/__init__.py @@ -11,15 +11,6 @@ Low level wrappers for the c++ api in ov::op. import numpy as np from openvino._pyopenvino.op import Constant - -"""Retrieve Constant inner data. - - Internally uses PyBind11 Numpy's buffer protocol. - - :return Numpy array containing internally stored constant data. -""" -Constant.get_data = lambda self: np.array(self, copy=True) - from openvino._pyopenvino.op import assign from openvino._pyopenvino.op import Parameter from openvino._pyopenvino.op import if_op diff --git a/src/bindings/python/src/openvino/runtime/opset13/__init__.py b/src/bindings/python/src/openvino/runtime/opset13/__init__.py index 80741ecd4af..fb36c096bc1 100644 --- a/src/bindings/python/src/openvino/runtime/opset13/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset13/__init__.py @@ -28,7 +28,7 @@ from openvino.runtime.opset1.ops import ceiling from openvino.runtime.opset1.ops import ceiling as ceil from openvino.runtime.opset1.ops import clamp from openvino.runtime.opset1.ops import concat -from openvino.runtime.opset1.ops import constant +from openvino.runtime.opset13.ops import constant from openvino.runtime.opset1.ops import convert from openvino.runtime.opset1.ops import convert_like from openvino.runtime.opset1.ops import convolution diff --git a/src/bindings/python/src/openvino/runtime/opset13/ops.py b/src/bindings/python/src/openvino/runtime/opset13/ops.py index a058e0526c2..f2495131eee 100644 --- a/src/bindings/python/src/openvino/runtime/opset13/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset13/ops.py @@ -4,15 +4,22 @@ """Factory functions for ops added to openvino opset13.""" from functools import partial -from typing import Optional +from typing import Optional, Union +import logging -from openvino.runtime import Node +import numpy as np + +log = logging.getLogger(__name__) + +from openvino.runtime import Node, Shape, Type +from openvino.runtime.op import Constant from openvino.runtime.opset_utils import _get_node_factory from openvino.runtime.utils.decorators import binary_op, nameable_op, unary_op from openvino.runtime.utils.types import ( + NumericData, NodeInput, + NumericType, as_nodes, - as_node, ) _get_node_factory_opset13 = partial(_get_node_factory, "opset13") @@ -222,3 +229,73 @@ def scaled_dot_product_attention( "causal": causal, } return _get_node_factory_opset13().create("ScaledDotProductAttention", inputs, attributes) + + +@nameable_op +def constant( + value: Union[NumericData, np.number, bool, np.bool_, list], + dtype: Union[NumericType, Type] = None, + name: Optional[str] = None, + *, + shared_memory: bool = False, +) -> Constant: + """Create a Constant node from provided value. + + :param value: One of: array of values or scalar to initialize node with. + :param dtype: The data type of provided data. + If dtype does not match, data will be converted. + Note: disables sharing of the memory when convertion occurs. + :param name: Optional name for output node. + :param shared_memory: keyword-only argument. + If `True`, this Constant's memory is being shared with a host, + that means the responsibility of keeping host memory is + on the side of a user. Any action performed on the host + memory is reflected on this Constant's memory! + If `False`, data is being copied to this Constant. + Requires data to be C_CONTIGUOUS if `True`. + Disabled by default if: + - value is a scalar. + - dtype is one of: Type.u1, Type.i4, Type.u4, Type.nf4, Type.bf16. + - dtype force conversion of data. + :return: The Constant node initialized with provided data. + """ + if isinstance(value, np.ndarray): + _value, _shared_memory = value, shared_memory + else: + _value, _shared_memory = np.array(value), False + log.warning(f"Converting scalar to corresponding type of {_value.dtype}. Memory sharing is disabled by default.") + # Handle type casting, when dtype is not None: + if dtype: + # Expect packed data, use different constructor to handle it correctly: + if dtype in [Type.u1, Type.i4, Type.u4, Type.nf4]: + if not np.allclose(_value, 0): + raise RuntimeError( + f"All values must be equal to 0 to initialize Constant with type of {dtype}. " + "Please use `openvino.helpers` module and `pack_data`, `unpack_data` functions to fill this Constant's data.") + log.warning(f"Constant initialized with packed type of {dtype}. Memory sharing is disabled by default.") + return Constant(dtype, Shape(_value.shape), _value.flatten().tolist()) + elif dtype in [Type.bf16]: + if not np.allclose(_value, 0): + raise RuntimeError( + f"All values must be equal to 0 to initialize Constant with type of {dtype}. " + "Please use `this_constant.data[:] = ...` to fill this Constant's data.") + log.warning(f"Constant initialized with OpenVINO custom {dtype}. Memory sharing is disabled by default.") + return Constant(dtype, Shape(_value.shape), _value.flatten().tolist()) + # General use-case for all other types: + else: + _dtype = dtype.to_dtype() if isinstance(dtype, Type) else dtype + if _dtype is int: + log.warning("Converting scalar type of undefined bitwidth to 32-bit integer. Memory sharing is disabled by default.") + _value, _shared_memory = _value.astype(np.int32), False + elif _dtype is float: + log.warning("Converting scalar type of undefined bitwidth to 32-bit float. Memory sharing is disabled by default.") + _value, _shared_memory = _value.astype(np.float32), False + elif _dtype is bool: + log.warning("Converting bool type to numpy bool. Memory sharing is disabled by default.") + _value, _shared_memory = _value.astype(np.bool_), False + else: + if _dtype != _value.dtype: + log.warning(f"Converting value of {_value.dtype} to {_dtype}. Memory sharing is disabled by default.") + _value, _shared_memory = _value.astype(_dtype), False + # Create Constant itself: + return Constant(_value, shared_memory=_shared_memory) diff --git a/src/bindings/python/src/pyopenvino/graph/ops/constant.cpp b/src/bindings/python/src/pyopenvino/graph/ops/constant.cpp index 92f5b0e2ace..69348f1f947 100644 --- a/src/bindings/python/src/pyopenvino/graph/ops/constant.cpp +++ b/src/bindings/python/src/pyopenvino/graph/ops/constant.cpp @@ -32,13 +32,15 @@ std::vector _get_strides(const ov::op::v0::Constant& self) { auto shape = self.get_shape(); if (element_type == ov::element::boolean) { return _get_byte_strides(shape); - } else if (element_type == ov::element::f16) { + } else if (element_type == ov::element::f16 || element_type == ov::element::bf16) { + // WA for bf16, returned as f16 array return _get_byte_strides(shape); } else if (element_type == ov::element::f32) { return _get_byte_strides(shape); } else if (element_type == ov::element::f64) { return _get_byte_strides(shape); - } else if (element_type == ov::element::i8) { + } else if (element_type == ov::element::i8 || element_type == ov::element::i4) { + // WA for i4, returned as int8 array return _get_byte_strides(shape); } else if (element_type == ov::element::i16) { return _get_byte_strides(shape); @@ -46,7 +48,9 @@ std::vector _get_strides(const ov::op::v0::Constant& self) { return _get_byte_strides(shape); } else if (element_type == ov::element::i64) { return _get_byte_strides(shape); - } else if (element_type == ov::element::u8 || element_type == ov::element::u1) { + } else if (element_type == ov::element::u8 || element_type == ov::element::u1 || element_type == ov::element::u4 || + element_type == ov::element::nf4) { + // WA for u1, u4, nf4, all returned as packed uint8 arrays return _get_byte_strides(shape); } else if (element_type == ov::element::u16) { return _get_byte_strides(shape); @@ -59,6 +63,7 @@ std::vector _get_strides(const ov::op::v0::Constant& self) { } } +// TODO: Remove in future and re-use `get_data` template py::buffer_info _get_buffer_info(const ov::op::v0::Constant& c) { ov::Shape shape = c.get_shape(); @@ -71,6 +76,7 @@ py::buffer_info _get_buffer_info(const ov::op::v0::Constant& c) { ); } +// TODO: Remove in future and re-use `get_data` template <> py::buffer_info _get_buffer_info(const ov::op::v0::Constant& c) { ov::Shape shape = c.get_shape(); @@ -160,6 +166,7 @@ void regclass_graph_op_Constant(py::module m) { } }); + // TODO: Remove in future and re-use `get_data` // Provide buffer access constant.def_buffer([](const ov::op::v0::Constant& self) -> py::buffer_info { auto element_type = self.get_element_type(); @@ -192,6 +199,26 @@ void regclass_graph_op_Constant(py::module m) { } }); + constant.def( + "get_data", + [](ov::op::v0::Constant& self) { + auto ov_type = self.get_element_type(); + auto dtype = Common::ov_type_to_dtype().at(ov_type); + if (ov_type.bitwidth() < Common::values::min_bitwidth) { + return py::array(dtype, self.get_byte_size(), self.get_data_ptr()); + } + return py::array(dtype, self.get_shape(), _get_strides(self), self.get_data_ptr()); + }, + R"( + Access to Constant's data - creates a copy of data. + + Returns numpy array with corresponding shape and dtype. + For Constants with openvino specific element type, such as u1, + it returns linear array, with uint8 / int8 numpy dtype. + + :rtype: numpy.array + )"); + constant.def_property_readonly( "data", [](ov::op::v0::Constant& self) { @@ -203,7 +230,7 @@ void regclass_graph_op_Constant(py::module m) { return py::array(dtype, self.get_shape(), _get_strides(self), self.get_data_ptr(), py::cast(self)); }, R"( - Access to Constant's data. + Access to Constant's data - creates a view of data. Returns numpy array with corresponding shape and dtype. For Constants with openvino specific element type, such as u1, diff --git a/src/bindings/python/tests/test_graph/test_constant.py b/src/bindings/python/tests/test_graph/test_constant.py new file mode 100644 index 00000000000..d8642c2144b --- /dev/null +++ b/src/bindings/python/tests/test_graph/test_constant.py @@ -0,0 +1,358 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np + +import openvino.runtime as ov +import openvino.runtime.opset13 as ops +from openvino.runtime import Type +from openvino.runtime.op import Constant +from openvino.helpers import pack_data, unpack_data + +import pytest +from enum import Enum + + +class DataGetter(Enum): + COPY = 1 + VIEW = 2 + + +@pytest.mark.parametrize( + ("src_dtype"), + [ + (np.float16), + (np.float32), + (np.float64), + (np.int8), + (np.uint8), + (np.int16), + (np.uint16), + (np.int32), + (np.uint32), + (np.int64), + (bool), + (np.bool_), + ], +) +@pytest.mark.parametrize( + ("dst_dtype"), + [ + (ov.Type.f32), + (ov.Type.f64), + (ov.Type.f16), + (ov.Type.i8), + (ov.Type.u8), + (ov.Type.i32), + (ov.Type.u32), + (ov.Type.i16), + (ov.Type.u16), + (ov.Type.i64), + (ov.Type.u64), + (ov.Type.boolean), + (np.float16), + (np.float32), + (np.float64), + (np.int8), + (np.uint8), + (np.int16), + (np.uint16), + (np.int32), + (np.uint32), + (np.int64), + (bool), + (np.bool_), + ], +) +@pytest.mark.parametrize( + ("shared_flag"), + [ + (True), + (False), + ], +) +@pytest.mark.parametrize( + ("data_getter"), + [ + (DataGetter.COPY), + (DataGetter.VIEW), + ], +) +def test_init_with_array(src_dtype, dst_dtype, shared_flag, data_getter): + data = np.random.rand(1, 2, 16, 8) + 0.5 + data = data.astype(src_dtype) + # C-style data for shared memory + if shared_flag is True: + data = np.ascontiguousarray(data) + + # Create constant from based on numpy dtype or openvino type + ov_const = ops.constant(data, dtype=dst_dtype, shared_memory=shared_flag) + + # Check shape and element type of Constant class + assert isinstance(ov_const, Constant) + assert np.all(tuple(ov_const.shape) == data.shape) + # Additionally check if Constant type matches dst_type if ov.Type was passed: + if isinstance(dst_dtype, Type): + assert ov_const.get_element_type() == dst_dtype + # Convert to dtype if OpenVINO Type + _dst_dtype = dst_dtype.to_dtype() if isinstance(dst_dtype, Type) else dst_dtype + assert ov_const.get_element_type().to_dtype() == _dst_dtype + # Cast original data and compare values to Constant + expected_result = data.astype(_dst_dtype) + if data_getter == DataGetter.COPY: + const_data = ov_const.get_data() + elif data_getter == DataGetter.VIEW: + const_data = ov_const.data + else: + raise AttributeError("Unknown DataGetter passed!") + assert const_data.dtype == _dst_dtype + assert np.allclose(const_data, expected_result) + + +@pytest.mark.parametrize( + ("init_value"), + [ + (1.5), # float + (2), # int + ], +) +@pytest.mark.parametrize( + ("src_dtype"), + [ + (float), + (int), + (np.float16), + (np.float32), + (np.float64), + (np.int8), + (np.uint8), + (np.int16), + (np.uint16), + (np.int32), + (np.uint32), + (np.int64), + (bool), + (np.bool_), + ], +) +@pytest.mark.parametrize( + ("dst_dtype"), + [ + (ov.Type.f32), + (ov.Type.f64), + (ov.Type.f16), + (ov.Type.i8), + (ov.Type.u8), + (ov.Type.i32), + (ov.Type.u32), + (ov.Type.i16), + (ov.Type.u16), + (ov.Type.i64), + (ov.Type.u64), + (ov.Type.boolean), + (np.float16), + (np.float32), + (np.float64), + (np.int8), + (np.uint8), + (np.int16), + (np.uint16), + (np.int32), + (np.uint32), + (np.int64), + (bool), + (np.bool_), + ], +) +@pytest.mark.parametrize( + ("shared_flag"), + [ + (True), + (False), + ], +) +@pytest.mark.parametrize( + ("data_getter"), + [ + (DataGetter.COPY), + (DataGetter.VIEW), + ], +) +def test_init_with_scalar(init_value, src_dtype, dst_dtype, shared_flag, data_getter): + data = src_dtype(init_value) + # Create constant from based on numpy dtype or openvino type + ov_const = ops.constant(data, dtype=dst_dtype, shared_memory=shared_flag) + + # Check shape and element type of Constant class + assert isinstance(ov_const, Constant) + assert np.all(list(ov_const.shape) == []) + # Additionally check if Constant type matches dst_type if ov.Type was passed: + if isinstance(dst_dtype, Type): + assert ov_const.get_element_type() == dst_dtype + # Convert to dtype if OpenVINO Type + _dst_dtype = dst_dtype.to_dtype() if isinstance(dst_dtype, Type) else dst_dtype + assert ov_const.get_element_type().to_dtype() == _dst_dtype + # Cast original data and compare values to Constant + expected_result = np.array(data).astype(_dst_dtype) + if data_getter == DataGetter.COPY: + const_data = ov_const.get_data() + elif data_getter == DataGetter.VIEW: + const_data = ov_const.data + else: + raise AttributeError("Unknown DataGetter passed!") + assert const_data.dtype == _dst_dtype + assert np.allclose(const_data, expected_result) + + +@pytest.mark.parametrize( + ("src_dtype"), + [ + (np.float16), + (np.uint16), + ], +) +@pytest.mark.parametrize( + ("shared_flag"), + [ + (True), + (False), + ], +) +@pytest.mark.parametrize( + ("data_getter"), + [ + (DataGetter.COPY), + (DataGetter.VIEW), + ], +) +def test_init_bf16(src_dtype, shared_flag, data_getter): + data = np.random.rand(1, 2, 16, 8) + 0.5 + data = data.astype(src_dtype) + + # To create bf16 constant, allocate memory and populate it: + ov_const = ops.constant(np.zeros(shape=data.shape, dtype=src_dtype), dtype=Type.bf16, shared_memory=shared_flag) + ov_const.data[:] = data + + # Check shape and element type of Constant class + assert isinstance(ov_const, Constant) + assert np.all(list(ov_const.shape) == [1, 2, 16, 8]) + assert ov_const.get_element_type() == Type.bf16 + + _dst_dtype = Type.bf16.to_dtype() + + assert ov_const.get_element_type().to_dtype() == _dst_dtype + # Compare values to Constant + if data_getter == DataGetter.COPY: + const_data = ov_const.get_data() + elif data_getter == DataGetter.VIEW: + const_data = ov_const.data + else: + raise AttributeError("Unknown DataGetter passed!") + assert const_data.dtype == _dst_dtype + assert np.allclose(const_data, data) + + +@pytest.mark.parametrize( + "shape", + [ + ([1, 3, 28, 28]), + ([1, 3, 27, 27]), + ], +) +@pytest.mark.parametrize( + ("low", "high", "ov_type", "src_dtype"), + [ + (0, 2, ov.Type.u1, np.uint8), + (0, 16, ov.Type.u4, np.uint8), + (-8, 7, ov.Type.i4, np.int8), + (0, 16, ov.Type.nf4, np.uint8), + ], +) +@pytest.mark.parametrize( + ("shared_flag"), + [ + (True), + (False), + ], +) +@pytest.mark.parametrize( + ("data_getter"), + [ + (DataGetter.COPY), + (DataGetter.VIEW), + ], +) +def test_constant_packing(shape, low, high, ov_type, src_dtype, shared_flag, data_getter): + data = np.random.uniform(low, high, shape).astype(src_dtype) + + # Allocate memory first: + ov_const = ops.constant(np.zeros(shape=data.shape, dtype=src_dtype), + dtype=ov_type, + shared_memory=shared_flag) + # Fill data with packed values + packed_data = pack_data(data, ov_const.get_element_type()) + ov_const.data[:] = packed_data + + # Always unpack the data! + if data_getter == DataGetter.COPY: + unpacked = unpack_data(ov_const.get_data(), ov_const.get_element_type(), ov_const.shape) + elif data_getter == DataGetter.VIEW: + unpacked = unpack_data(ov_const.data, ov_const.get_element_type(), ov_const.shape) + else: + raise AttributeError("Unknown DataGetter passed!") + + assert np.array_equal(unpacked, data) + + +@pytest.mark.parametrize( + ("shared_flag"), + [ + (True), + (False), + ], +) +def test_write_to_buffer(shared_flag): + arr_0 = np.ones([1, 3, 32, 32]) + ov_const = ops.constant(arr_0, shared_memory=shared_flag) + arr_1 = np.ones([1, 3, 32, 32]) + 1 + ov_const.data[:] = arr_1 + assert np.array_equal(ov_const.data, arr_1) + + +@pytest.mark.parametrize( + ("shared_flag"), + [ + (True), + (False), + ], +) +def test_memory_sharing(shared_flag): + arr = np.ones([1, 3, 32, 32]) + ov_const = ops.constant(arr, shared_memory=shared_flag) + arr += 1 + if shared_flag is True: + assert np.array_equal(ov_const.data, arr) + assert np.shares_memory(arr, ov_const.data) + else: + assert not np.array_equal(ov_const.data, arr) + assert not np.shares_memory(arr, ov_const.data) + + +@pytest.mark.parametrize( + ("ov_type", "src_dtype"), + [ + (Type.u1, np.uint8), + (Type.u4, np.uint8), + (Type.i4, np.int8), + (Type.nf4, np.uint8), + (Type.bf16, np.float16), + ], +) +def test_raise_for_packed_types(ov_type, src_dtype): + data = np.ones((2, 4, 16)).astype(src_dtype) + + with pytest.raises(RuntimeError) as err: + _ = ops.constant(data, dtype=ov_type) + + assert "All values must be equal to 0 to initialize Constant with type of" in str(err.value)