Torchfx backend (#18244)

* Added Torchscript Backend

* First commit for backend with Torch FX Decoder

* Merging changes from Torch FX branch

* Torch FX initial fixes (Temporary)

* Fixed type/shape issues in Torch FX decoder

* Added translation for built-in getitem

* MaxPool update & Output shape fix (Torch FX)

* Torch FX graph outputs fix

* Torch FX support for sigmoid and slu_

* Torch FX graph module caching

* Torch Fx partitioner cache removed

* Torch FX initial getitem replacer added

* Index check for torch fx getitem replacer

* Debug print removed from partitioner

* Added environment variables for pytorch tracing mode and openvino device

* FX translation fix for getitem & getitem replacer removed

* Added checks for PyTorch tracing mode environment variable

* Adding compile mode for fallback

* Added more ops for resnet18

* Added a check for environment variable

* Generalized addmm to work with torchscript and torchfx

* Added the missing batch_norm.default translation

* fx_backend: include get_attr ops to the partitions

* AddeTODO note t to improvget_attr algorithm

* created function for adding get_attr nodes

* fx_backend: added aten.mul.Tensor, re-enabled aten.empty.memory_format

* fx_backend: Additional op support/improvement for Inception V3

* Added comment for fix 64-bit to 32-bit max int conversion

* fx_backend: Update for avg_poolnd to support 3 inputs

* Fixed erorr in decoder.py

* TorchFX caching fix

* Torch backend, op support for Stable Diff. & BERT

* Arranged ops in order and added torch tensor mapping

* Added support for more ops for super glue

* TorchFX: Initial permanent fallback

* TorchFX: New ops for improved TorchVision support

* TorchFX backend optimizations for partitioning and tmp fallback

* working operator updates for superglue

* Updates to operators for superglue

* Removed max.dim and stack

* Cleanup

* Cleanup

* Fixed a couple of syntax issues

* Fixed a couple of syntax issues

* Added missing method to TorchFX Decoder

* Added missing method to TorchFX Decoder

* Removed redundant code for transpose

* TorchFX: Initial StableDiffusion support

* PyTorch decoder ovtype to ctype fix for int64

* Added ops for distilbert

* Fixed few unnecessary include statements

* Seperated TorchFX and TorchScript decoders

* Modified import statements to reflect two decoders

* f64 fix for TorchFX

* Import fix for PyTorch backend modules

* TorchFX serialize graph for debugging (Temporary)

* Serialize and load back feature enabled for TorchFX

* Temporary optimization to remove Broadcast

* Temporary SoftmaxRehapeElimination pass is added

* TorchFX custom model cache directory

* PyTorch bitwise translation, conversion checks enabled

* Naming fix in make_list_construct

* TorchFX: Added comments to Softmax and Slice translations

* translate_chunk temporarily removed for TS backend

* Fixed linter issues

* Addressed clang formatting issues

* Fixed few more clang and linter issues

* Fixed tests to use ts_decoder

* Fixed naming convention issues

* Added missing import

* Added inlined_inputs to TorchScriptDecoder

* Added tests for torch fx backend

* Removed magic numbers in PyTorch decoder utils

* TorchFX decoder data type fix

* Added cast from size_t to int

* TorchFX output handling code cleanup

* TorchFX: Use detached input tensor

* Added missing cast from size_t to int

* Added static cast in group_norm

* Fixed casting issue in split

---------

Co-authored-by: ynimmaga <yamini.nimmagadda@intel.com>
Co-authored-by: Cavus Mustafa <mustafa.cavus@intel.com>
This commit is contained in:
Surya Siddharth Pemmaraju
2023-07-26 07:23:42 -07:00
committed by GitHub
parent 1ee5b6dd3f
commit d42b53c070
60 changed files with 1881 additions and 234 deletions

View File

@@ -37,7 +37,7 @@ def get_traced_model(model, inputs=[], frozen=True):
@pytest.mark.precommit
def test_pytorch_decoder_get_output_type_str():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.py_pytorch_frontend import _Type as DecoderType
model = get_scripted_model(AtenDiv("trunc"))
@@ -53,7 +53,7 @@ def test_pytorch_decoder_get_output_type_str():
@pytest.mark.precommit
def test_pytorch_decoder_get_output_type_none():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.py_pytorch_frontend import _Type as DecoderType
model = get_scripted_model(AtenDiv(None))
@@ -69,7 +69,7 @@ def test_pytorch_decoder_get_output_type_none():
@pytest.mark.precommit
def test_pytorch_decoder_get_input_type_str():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.py_pytorch_frontend import _Type as DecoderType
model = get_scripted_model(AtenDiv("trunc"))
@@ -83,7 +83,7 @@ def test_pytorch_decoder_get_input_type_str():
@pytest.mark.precommit
def test_pytorch_decoder_get_input_type_none():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.py_pytorch_frontend import _Type as DecoderType
model = get_scripted_model(AtenDiv(None))
@@ -97,7 +97,7 @@ def test_pytorch_decoder_get_input_type_none():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_fp16_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -119,7 +119,7 @@ def test_pytorch_decoder_can_convert_fp16_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_bf16_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -141,7 +141,7 @@ def test_pytorch_decoder_can_convert_bf16_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_fp32_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -163,7 +163,7 @@ def test_pytorch_decoder_can_convert_fp32_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_fp64_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -185,7 +185,7 @@ def test_pytorch_decoder_can_convert_fp64_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_bool_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -207,7 +207,7 @@ def test_pytorch_decoder_can_convert_bool_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_u8_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -229,7 +229,7 @@ def test_pytorch_decoder_can_convert_u8_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_i8_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -251,7 +251,7 @@ def test_pytorch_decoder_can_convert_i8_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_i32_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -273,7 +273,7 @@ def test_pytorch_decoder_can_convert_i32_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_i64_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -295,7 +295,7 @@ def test_pytorch_decoder_can_convert_i64_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_int64_max():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
class I64MaxConst(torch.nn.Module):
def forward(self):
@@ -313,7 +313,7 @@ def test_pytorch_decoder_can_convert_int64_max():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_int_list():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class ListConst(torch.nn.Module):
@@ -336,7 +336,7 @@ def test_pytorch_decoder_can_convert_int_list():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_float_list():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class ListConst(torch.nn.Module):
@@ -359,7 +359,7 @@ def test_pytorch_decoder_can_convert_float_list():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_bool_list():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class ListConst(torch.nn.Module):
@@ -382,7 +382,7 @@ def test_pytorch_decoder_can_convert_bool_list():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_int_tuple():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class ListConst(torch.nn.Module):
@@ -405,7 +405,7 @@ def test_pytorch_decoder_can_convert_int_tuple():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_float_tuple():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class ListConst(torch.nn.Module):
@@ -428,7 +428,7 @@ def test_pytorch_decoder_can_convert_float_tuple():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_bool_tuple():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class ListConst(torch.nn.Module):
@@ -451,7 +451,7 @@ def test_pytorch_decoder_can_convert_bool_tuple():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_empty_list():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class aten_roll(torch.nn.Module):
@@ -479,7 +479,7 @@ def test_pytorch_decoder_can_convert_empty_list():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_int_scalar_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -510,7 +510,7 @@ def test_pytorch_decoder_can_convert_int_scalar_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_float_scalar_tensor():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
class SomeTensor(torch.nn.Module):
@@ -541,7 +541,7 @@ def test_pytorch_decoder_can_convert_float_scalar_tensor():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_tensor_list():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.runtime import PartialShape, Type
from typing import List, Optional
@@ -579,7 +579,7 @@ def test_pytorch_decoder_can_convert_tensor_list():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_tensor_list_empty():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from typing import List, Optional
class SomeTensor(torch.nn.Module):
@@ -607,7 +607,7 @@ def test_pytorch_decoder_can_convert_tensor_list_empty():
@pytest.mark.precommit
def test_pytorch_decoder_can_convert_optional_tensor_none():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from typing import Optional
class SomeTensor(torch.nn.Module):

View File

@@ -30,7 +30,7 @@ def get_scripted_model(model):
def test_pytorch_fe_set_input_shape():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
model = get_scripted_model(aten_relu())
decoder = TorchScriptPythonDecoder(model)
@@ -45,7 +45,7 @@ def test_pytorch_fe_set_input_shape():
def test_pytorch_fe_set_input_type():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
model = get_scripted_model(aten_relu())
decoder = TorchScriptPythonDecoder(model)
@@ -59,7 +59,7 @@ def test_pytorch_fe_set_input_type():
def test_pytorch_fe_set_input_value():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
model = get_scripted_model(aten_relu())
decoder = TorchScriptPythonDecoder(model)
@@ -75,7 +75,7 @@ def test_pytorch_fe_set_input_value():
def test_conversion_extension():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
class Model(torch.nn.Module):
def __init__(self):
@@ -185,7 +185,7 @@ def get_builtin_extensions_path():
def test_op_extension():
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
class Elu(torch.nn.Module):
def __init__(self, alpha):
@@ -216,7 +216,7 @@ def test_op_extension():
def test_pytorch_telemetry():
from openvino.frontend import TelemetryExtension
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
class MockTelemetry:
def __init__(self, stat):

View File

@@ -3,4 +3,5 @@ markers =
nightly
precommit
precommit_ts_backend
precommit_fx_backend
timeout

View File

@@ -8,7 +8,7 @@ import os
import numpy as np
from common.constants import test_device, test_precision
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from openvino.frontend import FrontEndManager
from openvino.runtime import Core, Type, PartialShape
@@ -65,13 +65,17 @@ class PytorchLayerTest:
else:
custom_eps = 1e-4
def use_ts_backend():
return(os.environ.get('USE_TS_BACKEND', False))
def use_torch_compile_backend():
torch_compile_env = os.getenv("PYTORCH_TRACING_MODE")
if torch_compile_env is not None:
if (torch_compile_env == "TORCHFX" or torch_compile_env == "TORCHSCRIPT"):
return True
return False
ov_inputs = flattenize_inputs(inputs)
if use_ts_backend():
self.ts_backend_test(model, torch_inputs, custom_eps)
if use_torch_compile_backend():
self.torch_compile_backend_test(model, torch_inputs, custom_eps)
else:
with torch.no_grad():
model.eval()
@@ -218,7 +222,7 @@ class PytorchLayerTest:
om.validate_nodes_and_infer_types()
return om
def ts_backend_test(self, model, inputs, custom_eps):
def torch_compile_backend_test(self, model, inputs, custom_eps):
torch._dynamo.reset()
with torch.no_grad():
model.eval()

View File

@@ -33,6 +33,7 @@ class TestAdaptiveAvgPool3D(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
def test_adaptive_avg_pool3d(self, ie_device, precision, ir_version, input_tensor, output_size):
self.input_tensor = input_tensor
self._test(*self.create_model(output_size), ie_device, precision, ir_version)

View File

@@ -47,6 +47,7 @@ class TestAdaptiveMaxPool2D(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
def test_adaptive_max_pool2d(self, ie_device, precision, ir_version, input_tensor, output_size, return_indices):
self.input_tensor = input_tensor
self._test(*self.create_model(output_size, return_indices), ie_device, precision, ir_version)

View File

@@ -39,6 +39,7 @@ class TestAdd(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
@pytest.mark.parametrize("op_type", ["add", "add_"])
def test_add(self, ie_device, precision, ir_version, alpha, input_rhs, op_type):
self.input_rhs = input_rhs
@@ -100,6 +101,7 @@ class TestAddTypes(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
def test_add_types(self, ie_device, precision, ir_version, lhs_type, lhs_shape, rhs_type, rhs_shape):
self.lhs_type = lhs_type
self.lhs_shape = lhs_shape
@@ -123,5 +125,6 @@ class TestAddLists(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
def test_add(self, ie_device, precision, ir_version):
self._test(*self.create_model(), ie_device, precision, ir_version)

View File

@@ -58,6 +58,7 @@ class TestBatchNorm(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
def test_batch_norm(self, weights, bias, eps, train, running_stats, ie_device, precision, ir_version, kwargs_to_prepare_input):
self._test(*self.create_model(weights, bias, eps, train, running_stats),
ie_device, precision, ir_version, kwargs_to_prepare_input=kwargs_to_prepare_input, dynamic_shapes=False, use_mo_convert=False)

View File

@@ -3,7 +3,7 @@
import pytest
from openvino.frontend import FrontEndManager
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from pytorch_layer_test_class import PytorchLayerTest
@@ -165,7 +165,7 @@ class TestConv2DInSubgraph(PytorchLayerTest):
def _prepare_input(self):
import numpy as np
return (np.random.randn(2, 3, 25, 25).astype(np.float32), np.array([1], dtype=np.int32))
def convert_directly_via_frontend(self, model, example_input, trace_model, dynamic_shapes, ov_inputs, freeze_model):
# Overload function to allow reproduction of issue caused by additional freeze.
import torch

View File

@@ -208,6 +208,7 @@ class TestConvolution(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
def test_convolution1d(self, params, bias, underscore, ie_device, precision, ir_version):
self._test(*self.create_model(**params, bias=bias, underscore=underscore),
ie_device, precision, ir_version, dynamic_shapes=params['groups'] == 1,
@@ -219,6 +220,7 @@ class TestConvolution(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
def test_convolution2d(self, params, bias, underscore, ie_device, precision, ir_version):
self._test(*self.create_model(**params, bias=bias, underscore=underscore),
ie_device, precision, ir_version, dynamic_shapes=params['groups'] == 1)

View File

@@ -6,7 +6,7 @@ import numpy as np
import torch
from openvino.frontend import FrontEndManager
from openvino.frontend.pytorch.decoder import TorchScriptPythonDecoder
from openvino.frontend.pytorch.ts_decoder import TorchScriptPythonDecoder
from pytorch_layer_test_class import PytorchLayerTest

View File

@@ -97,6 +97,7 @@ class TestRemainderTypes(PytorchLayerTest):
@pytest.mark.nightly
@pytest.mark.precommit
@pytest.mark.precommit_ts_backend
@pytest.mark.precommit_fx_backend
def test_remainder_types(self, ie_device, precision, ir_version, lhs_type, lhs_shape, rhs_type, rhs_shape):
self.lhs_type = lhs_type
self.lhs_shape = lhs_shape