IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Telemetry for IE dependency warnings (#4431)

Browse Source 
* Initial telemetry for IE dependency warnings

* Unified MO/IE version extraction; updated telemetry messages to use simplified versions

* Update telemetry

* Renamings; improvements; comments

* Turn On telemetry

* Added tests

* Added versions_mismatch flag

* plarform -> system
This commit is contained in:
Gleb Kazantaev
2021-03-01 13:30:40 +03:00
committed by GitHub
parent 5f14fe9ca1
commit 1bdccb41df
10 changed files with 232 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
model-optimizer/automation/package_BOM.txt
View File

@@ -980,6 +980,7 @@ mo/utils/find_inputs.py
mo/utils/get_ov_update_message.py
mo/utils/graph.py
mo/utils/guess_framework.py
mo/utils/ie_version.py
mo/utils/import_extensions.py
mo/utils/ir_engine/__init__.py
mo/utils/ir_engine/compare_graphs.py

29
model-optimizer/mo/main.py
View File

@@ -19,6 +19,7 @@ import datetime
import logging as log
import os
import sys
import platform
import subprocess
import traceback
from collections import OrderedDict
@@ -35,13 +36,13 @@ from mo.utils import import_extensions
from mo.utils.cli_parser import get_placeholder_shapes, get_tuple_values, get_model_name, \
get_common_cli_options, get_caffe_cli_options, get_tf_cli_options, get_mxnet_cli_options, get_kaldi_cli_options, \
get_onnx_cli_options, get_mean_scale_dictionary, parse_tuple_pairs, get_freeze_placeholder_values, get_meta_info
from mo.utils.error import Error, FrameworkError
from mo.utils.error import Error, FrameworkError, classify_error_type
from mo.utils.get_ov_update_message import get_ov_update_message
from mo.utils.guess_framework import deduce_framework_by_namespace
from mo.utils.logger import init_logger
from mo.utils.model_analysis import AnalysisResults
from mo.utils.utils import refer_to_faq_msg
from mo.utils.version import get_version
from mo.utils.version import get_version, get_simplified_mo_version, get_simplified_ie_version
from mo.utils.versions_checker import check_requirements
from mo.utils.find_ie_version import find_ie_version
@@ -158,7 +159,6 @@ def prepare_ir(argv: argparse.Namespace):
# If the IE was not found, it will not print the MO version, so we have to print it manually
print("{}: \t{}".format("Model Optimizer version", get_version()))
except Exception as e:
# TODO: send exception message
pass
ret_code = check_requirements(framework=argv.framework)
@@ -270,19 +270,30 @@ def emit_ir(graph: Graph, argv: argparse.Namespace):
output_dir = argv.output_dir if argv.output_dir != '.' else os.getcwd()
orig_model_name = os.path.normpath(os.path.join(output_dir, argv.model_name))
return_code = "not executed"
# This try-except is additional reinsurance that the IE
# dependency search does not break the MO pipeline
try:
if find_ie_version(silent=True):
path_to_offline_transformations = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'back',
'offline_transformations.py')
status = subprocess.run([sys.executable, path_to_offline_transformations, orig_model_name], env=os.environ, timeout=100)
if status.returncode != 0 and not argv.silent:
print("[ WARNING ] offline_transformations return code {}".format(status.returncode))
status = subprocess.run([sys.executable, path_to_offline_transformations, orig_model_name], env=os.environ, timeout=10)
return_code = status.returncode
if return_code != 0 and not argv.silent:
print("[ WARNING ] offline_transformations return code {}".format(return_code))
except Exception as e:
# TODO: send error message
pass
message = str(dict({
"platform": platform.system(),
"mo_version": get_simplified_mo_version(),
"ie_version": get_simplified_ie_version(env=os.environ),
"python_version": sys.version,
"return_code": return_code
}))
t = tm.Telemetry()
t.send_event('mo', 'offline_transformations_status', message)
print('[ SUCCESS ] Generated IR version {} model.'.format(get_ir_version(argv)))
print('[ SUCCESS ] XML file: {}.xml'.format(orig_model_name))
print('[ SUCCESS ] BIN file: {}.bin'.format(orig_model_name))
@@ -316,9 +327,9 @@ def driver(argv: argparse.Namespace):
def main(cli_parser: argparse.ArgumentParser, framework: str):
telemetry = tm.Telemetry(app_name='Model Optimizer', app_version=get_version())
telemetry = tm.Telemetry(app_name='Model Optimizer', app_version=get_simplified_mo_version())
telemetry.start_session()
telemetry.send_event('mo', 'version', get_version())
telemetry.send_event('mo', 'version', get_simplified_mo_version())
try:
# Initialize logger with 'ERROR' as default level to be able to form nice messages
# before arg parser deliver log_level requested by user

75
model-optimizer/mo/utils/check_ie_bindings.py
View File

@@ -18,15 +18,30 @@
import os
import re
import sys
import argparse
import platform
try:
# needed by find_ie_version.py which call check_ie_bindings.py as python script
import version # pylint: disable=import-error
except ImportError:
import mo.utils.version
import mo
execution_type = "mo"
except ModuleNotFoundError:
mo_root_path = os.path.normpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
sys.path.insert(0, mo_root_path)
execution_type = "install_prerequisites.{}".format("bat" if platform.system() == "Windows" else "sh")
from extract_release_version import extract_release_version
import mo.utils.version as v
import telemetry.telemetry as tm
from mo.utils.error import classify_error_type
def send_telemetry(mo_version: str, message: str, event_type: str):
t = tm.Telemetry(app_name='Model Optimizer', app_version=mo_version)
t.start_session()
t.send_event(execution_type, event_type, message)
t.end_session()
t.force_shutdown(1.0)
def import_core_modules(silent: bool, path_to_module: str):
@@ -36,26 +51,42 @@ def import_core_modules(silent: bool, path_to_module: str):
import openvino # pylint: disable=import-error
if silent:
return True
ie_version = str(get_version())
mo_version = str(version.get_version()) # pylint: disable=no-member
mo_version = str(v.get_version()) # pylint: disable=no-member
if not silent:
print("\t- {}: \t{}".format("Inference Engine found in", os.path.dirname(openvino.__file__)))
print("{}: \t{}".format("Inference Engine version", ie_version))
print("{}: \t {}".format("Model Optimizer version", mo_version))
print("\t- {}: \t{}".format("Inference Engine found in", os.path.dirname(openvino.__file__)))
print("{}: \t{}".format("Inference Engine version", ie_version))
print("{}: \t {}".format("Model Optimizer version", mo_version))
versions_mismatch = False
# MO and IE version have a small difference in the beginning of version because
# IE version also includes API version. For example:
# Inference Engine version: 2.1.custom_HEAD_4c8eae0ee2d403f8f5ae15b2c9ad19cfa5a9e1f9
# Model Optimizer version: custom_HEAD_4c8eae0ee2d403f8f5ae15b2c9ad19cfa5a9e1f9
# So to match this versions we skip IE API version.
if not re.match(r"^([0-9]+).([0-9]+).{}$".format(mo_version), ie_version):
extracted_release_version = extract_release_version()
is_custom_mo_version = extracted_release_version == (None, None)
if not silent:
print("[ WARNING ] Model Optimizer and Inference Engine versions do no match.")
print("[ WARNING ] Consider building the Inference Engine Python API from sources or reinstall OpenVINO (TM) toolkit using \"pip install openvino{}\" {}".format(
"", "(may be incompatible with the current Model Optimizer version)" if is_custom_mo_version else "=={}.{}".format(*extracted_release_version), ""))
versions_mismatch = True
extracted_mo_release_version = v.extract_release_version(mo_version)
mo_is_custom = extracted_mo_release_version == (None, None)
print("[ WARNING ] Model Optimizer and Inference Engine versions do no match.")
print("[ WARNING ] Consider building the Inference Engine Python API from sources or reinstall OpenVINO (TM) toolkit using", end=" ")
if mo_is_custom:
print("\"pip install openvino\" (may be incompatible with the current Model Optimizer version)")
else:
print("\"pip install openvino=={}.{}\"".format(*extracted_mo_release_version))
simplified_mo_version = v.get_simplified_mo_version()
message = str(dict({
"platform": platform.system(),
"mo_version": simplified_mo_version,
"ie_version": v.get_simplified_ie_version(version=ie_version),
"versions_mismatch": versions_mismatch,
}))
send_telemetry(simplified_mo_version, message, 'ie_version_check')
return True
except Exception as e:
@@ -63,6 +94,18 @@ def import_core_modules(silent: bool, path_to_module: str):
if "No module named 'openvino'" not in str(e) and not silent:
print("[ WARNING ] Failed to import Inference Engine Python API in: {}".format(path_to_module))
print("[ WARNING ] {}".format(e))
# Send telemetry message about warning
simplified_mo_version = v.get_simplified_mo_version()
message = str(dict({
"platform": platform.system(),
"mo_version": simplified_mo_version,
"ie_version": v.get_simplified_ie_version(env=os.environ),
"python_version": sys.version,
"error_type": classify_error_type(e),
}))
send_telemetry(simplified_mo_version, message, 'ie_import_failed')
return False

17
model-optimizer/mo/utils/error.py
View File

@@ -1,5 +1,5 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Copyright (C) 2018-2021 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import re
class BasicError(Exception):
@@ -44,3 +45,17 @@ class InternalError(BasicError):
""" Not user-friendly error: user cannot fix it and it points to the bug inside MO. """
pass
def classify_error_type(e):
patterns = [
# Example: No module named 'openvino.offline_transformations.offline_transformations_api'
r"No module named \'\S+\'",
# Example: cannot import name 'IECore' from 'openvino.inference_engine' (unknown location)
r"cannot import name \'\S+\'",
]
error_message = str(e)
for pattern in patterns:
m = re.search(pattern, error_message)
if m:
return m.group(0)
return "undefined"

37
model-optimizer/mo/utils/error_test.py Normal file
View File

@@ -0,0 +1,37 @@
"""
Copyright (C) 2021 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import unittest
from mo.utils.error import classify_error_type
class TestingErrorClassifier(unittest.TestCase):
def test_no_module(self):
message = "No module named 'openvino.offline_transformations.offline_transformations_api'"
self.assertEqual(classify_error_type(message), message)
def test_no_module_neg(self):
message = "No module 'openvino'"
self.assertEqual(classify_error_type(message), "undefined")
def test_cannot_import_name(self):
message = "cannot import name 'IECore' from 'openvino.inference_engine' (unknown location)"
self.assertEqual(classify_error_type(message), "cannot import name 'IECore'")
def test_cannot_import_name_neg(self):
message = "import name 'IECore' from 'openvino.inference_engine' (unknown location)"
self.assertEqual(classify_error_type(message), "undefined")

23
model-optimizer/mo/utils/extract_release_version.py
View File

@@ -13,30 +13,13 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import re
try:
# needed by install_prerequisites which call extract_release_version as python script
from version import get_version
from version import extract_release_version, get_version
except ImportError:
from mo.utils.version import get_version
def extract_release_version():
version = get_version()
patterns = [
# captures release version set by CI for example: '2021.1.0-1028-55e4d5673a8'
r"^([0-9]+).([0-9]+)*",
# captures release version generated by MO from release branch, for example: 'custom_releases/2021/1_55e4d567'
r"_releases/([0-9]+)/([0-9]+)_*"
]
for pattern in patterns:
m = re.search(pattern, version)
if m and len(m.groups()) == 2:
return m.group(1), m.group(2)
return None, None
from mo.utils.version import extract_release_version, get_version
if __name__ == "__main__":
print("{}.{}".format(*extract_release_version()))
print("{}.{}".format(*extract_release_version(get_version())))

27
model-optimizer/mo/utils/ie_version.py Normal file
View File

@@ -0,0 +1,27 @@
"""
Copyright (C) 2021 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
def get_ie_version():
try:
from openvino.inference_engine import get_version # pylint: disable=import-error
return get_version()
except:
return None
if __name__ == "__main__":
print(get_ie_version())

40
model-optimizer/mo/utils/version.py
View File

@@ -14,6 +14,8 @@
limitations under the License.
"""
import os
import re
import sys
import subprocess
@@ -41,3 +43,41 @@ def get_version():
with open(version_txt) as f:
version = f.readline().replace('\n', '')
return version
def extract_release_version(version: str):
patterns = [
# captures release version set by CI for example: '2021.1.0-1028-55e4d5673a8'
r"^([0-9]+).([0-9]+)*",
# captures release version generated by MO from release branch, for example: 'custom_releases/2021/1_55e4d567'
r"_releases/([0-9]+)/([0-9]+)_*"
]
for pattern in patterns:
m = re.search(pattern, version)
if m and len(m.groups()) == 2:
return m.group(1), m.group(2)
return None, None
def simplify_version(version: str):
release_version = extract_release_version(version)
if release_version == (None, None):
return "custom"
return "{}.{}".format(*release_version)
def get_simplified_mo_version():
return simplify_version(get_version())
def get_simplified_ie_version(env=dict(), version=None):
if version is None:
try:
version = subprocess.check_output([sys.executable, os.path.join(os.path.dirname(__file__), "ie_version.py")], timeout=2, env=env).strip().decode()
except:
return "ie not found"
m = re.match(r"^([0-9]+).([0-9]+).(.*)", version)
if m and len(m.groups()) == 3:
return simplify_version(m.group(3))
return "custom"

34
model-optimizer/mo/utils/version_test.py
View File

@@ -1,5 +1,5 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Copyright (C) 2018-2021 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -19,8 +19,7 @@ import unittest.mock as mock
from unittest.mock import mock_open
from unittest.mock import patch
from mo.utils.version import get_version
from mo.utils.extract_release_version import extract_release_version
from mo.utils.version import get_version, extract_release_version, get_simplified_ie_version, get_simplified_mo_version
class TestingVersion(unittest.TestCase):
@@ -36,18 +35,41 @@ class TestingVersion(unittest.TestCase):
def test_release_version_extractor(self, mock_open, mock_isfile):
mock_isfile.return_value = True
mock_open.return_value.__enter__ = mock_open
self.assertEqual(extract_release_version(), ('2021', '1'))
self.assertEqual(extract_release_version(get_version()), ('2021', '1'))
@patch('os.path.isfile')
@mock.patch('builtins.open', new_callable=mock_open, create=True, read_data='custom_releases/2021/1_55e4d5673a8')
def test_custom_release_version_extractor(self, mock_open, mock_isfile):
mock_isfile.return_value = True
mock_open.return_value.__enter__ = mock_open
self.assertEqual(extract_release_version(), ('2021', '1'))
self.assertEqual(extract_release_version(get_version()), ('2021', '1'))
@patch('os.path.isfile')
@mock.patch('builtins.open', new_callable=mock_open, create=True, read_data='custom_my_branch/fix_55e4d5673a8')
def test_release_version_extractor_neg(self, mock_open, mock_isfile):
mock_isfile.return_value = True
mock_open.return_value.__enter__ = mock_open
self.assertEqual(extract_release_version(), (None, None))
self.assertEqual(extract_release_version(get_version()), (None, None))
@patch('os.path.isfile')
@mock.patch('builtins.open', new_callable=mock_open, create=True, read_data='custom_releases/2021/1_55e4d5673a8')
def test_simplify_mo_version_release(self, mock_open, mock_isfile):
mock_isfile.return_value = True
mock_open.return_value.__enter__ = mock_open
self.assertEqual(get_simplified_mo_version(), "2021.1")
@patch('os.path.isfile')
@mock.patch('builtins.open', new_callable=mock_open, create=True, read_data='custom_my_branch/fix_55e4d5673a8')
def test_simplify_mo_version_custom(self, mock_open, mock_isfile):
mock_isfile.return_value = True
mock_open.return_value.__enter__ = mock_open
self.assertEqual(get_simplified_mo_version(), "custom")
def test_simplify_ie_version_release(self):
self.assertEqual(get_simplified_ie_version(version="2.1.custom_releases/2021/3_4c8eae"), "2021.3")
def test_simplify_ie_version_release_neg(self):
self.assertEqual(get_simplified_ie_version(version="custom_releases/2021/3_4c8eae"), "custom")
def test_simplify_ie_version_custom(self):
self.assertEqual(get_simplified_ie_version(version="2.1.custom_my/branch/3_4c8eae"), "custom")

4
model-optimizer/telemetry/telemetry.py
View File

@@ -40,9 +40,7 @@ class Telemetry(metaclass=SingletonMetaClass):
if not hasattr(self, 'tid'):
self.tid = None
if app_name is not None:
# temporary disable telemetry
# self.consent = isip.isip_consent() == isip.ISIPConsent.APPROVED
self.consent = False
self.consent = isip.isip_consent() == isip.ISIPConsent.APPROVED
# override default tid
if tid is not None:
self.tid = tid