From 48c9eaba565567e95e60f1db497cc5d42ad04342 Mon Sep 17 00:00:00 2001 From: Artyom Anokhov Date: Mon, 19 Jul 2021 18:38:54 +0300 Subject: [PATCH] Migrate Deployment Manager tool to Open Source (#6692) * Migrate Deployment Manager tool to Open Source * scripts/CMakeLists.txt: Added install rules for deployment manager --- scripts/CMakeLists.txt | 6 + .../deployment_manager/configs/darwin.json | 106 +++++++ scripts/deployment_manager/configs/linux.json | 145 +++++++++ .../deployment_manager/configs/windows.json | 140 +++++++++ .../deployment_manager/deployman/config.py | 97 ++++++ .../deployment_manager/deployman/logger.py | 39 +++ scripts/deployment_manager/deployman/main.py | 219 +++++++++++++ scripts/deployment_manager/deployman/ui.py | 288 ++++++++++++++++++ .../deployment_manager/deployment_manager.py | 25 ++ 9 files changed, 1065 insertions(+) create mode 100644 scripts/deployment_manager/configs/darwin.json create mode 100644 scripts/deployment_manager/configs/linux.json create mode 100644 scripts/deployment_manager/configs/windows.json create mode 100644 scripts/deployment_manager/deployman/config.py create mode 100644 scripts/deployment_manager/deployman/logger.py create mode 100644 scripts/deployment_manager/deployman/main.py create mode 100644 scripts/deployment_manager/deployman/ui.py create mode 100644 scripts/deployment_manager/deployment_manager.py diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index ac094ce648b..a54b5f97f3b 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -47,6 +47,12 @@ if(UNIX) COMPONENT install_dependencies) endif() +# install DeploymentManager +ie_cpack_add_component(deployment_manager REQUIRED) +install(DIRECTORY deployment_manager/ + DESTINATION deployment_tools/tools/deployment_manager + COMPONENT deployment_manager) + # install files for demo ie_cpack_add_component(demo_scripts DEPENDS core) diff --git a/scripts/deployment_manager/configs/darwin.json b/scripts/deployment_manager/configs/darwin.json new file mode 100644 index 00000000000..1ef4e8e443d --- /dev/null +++ b/scripts/deployment_manager/configs/darwin.json @@ -0,0 +1,106 @@ +{ + "version": "0.2", + "components": { + "setupvars": { + "mandatory" : "yes", + "files": [ + "bin" + ] + }, + "openvino_license": { + "mandatory" : "yes", + "files": [ + "licensing" + ] + }, + "ie_core": { + "group": ["ie"], + "files": [ + "deployment_tools/inference_engine/version.txt", + "deployment_tools/inference_engine/lib/intel64/libinference_engine.dylib", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_lp_transformations.dylib", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_transformations.dylib", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_preproc.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_ir_reader.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_onnx_reader.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_c_api.dylib", + "deployment_tools/inference_engine/lib/intel64/libAutoPlugin.so", + "deployment_tools/inference_engine/lib/intel64/plugins.xml", + "deployment_tools/inference_engine/external/tbb", + "deployment_tools/ngraph/lib" + ] + }, + "cpu": { + "ui_name": "Inference Engine Runtime for Intel(R) CPU", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/lib/intel64/libMKLDNNPlugin.so" + ] + }, + "vpu": { + "ui_name": "Inference Engine Runtime for Intel(R) Movidius(tm) VPU", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/lib/intel64/libmyriadPlugin.so", + "deployment_tools/inference_engine/lib/intel64/myriad_compile", + "deployment_tools/inference_engine/lib/intel64/myriad_perfcheck", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_legacy.dylib", + "deployment_tools/inference_engine/lib/intel64/usb-ma2x8x.mvcmd", + "deployment_tools/inference_engine/lib/intel64/pcie-ma2x8x.mvcmd" + ] + }, + "opencv": { + "ui_name": "OpenCV", + "group": ["opencv"], + "dependencies" : [], + "files": [ + "opencv/version.txt", + "opencv/setupvars.sh", + "opencv/lib", + "opencv/bin" + ] + }, + "python_common": { + "group": ["python"], + "dependencies" : ["ie_core"], + "files": [ + "python/python3", + "python/requirements.txt" + ] + }, + "python3.6": { + "ui_name": "OpenVINO Python API for Python3.6", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.6" + ] + }, + "python3.7": { + "ui_name": "OpenVINO Python API for Python3.7", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.7" + ] + }, + "python3.8": { + "ui_name": "OpenVINO Python API for Python3.8", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.8" + ] + }, + "python3.9": { + "ui_name": "OpenVINO Python API for Python3.9", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.9" + ] + } + } +} diff --git a/scripts/deployment_manager/configs/linux.json b/scripts/deployment_manager/configs/linux.json new file mode 100644 index 00000000000..e832ab933a0 --- /dev/null +++ b/scripts/deployment_manager/configs/linux.json @@ -0,0 +1,145 @@ +{ + "version": "0.2", + "components": { + "setupvars": { + "mandatory" : "yes", + "files": [ + "bin" + ] + }, + "openvino_dependencies": { + "mandatory" : "yes", + "files": [ + "install_dependencies/install_openvino_dependencies.sh" + ] + }, + "openvino_license": { + "mandatory" : "yes", + "files": [ + "licensing" + ] + }, + "ie_core": { + "group": ["ie"], + "files": [ + "deployment_tools/inference_engine/version.txt", + "deployment_tools/inference_engine/lib/intel64/libinference_engine.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_lp_transformations.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_transformations.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_preproc.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_ir_reader.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_onnx_reader.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_c_api.so", + "deployment_tools/inference_engine/lib/intel64/libAutoPlugin.so", + "deployment_tools/inference_engine/lib/intel64/plugins.xml", + "deployment_tools/inference_engine/external/tbb", + "deployment_tools/ngraph/lib" + ] + }, + "cpu": { + "ui_name": "Inference Engine Runtime for Intel(R) CPU", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/lib/intel64/libMKLDNNPlugin.so" + ] + }, + "gpu": { + "ui_name": "Inference Engine Runtime for Intel(R) Processor Graphics", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/lib/intel64/cache.json", + "deployment_tools/inference_engine/lib/intel64/libclDNNPlugin.so", + "install_dependencies/install_NEO_OCL_driver.sh" + ] + }, + "vpu": { + "ui_name": "Inference Engine Runtime for Intel(R) Movidius(tm) VPU", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/external/97-myriad-usbboot.rules", + "deployment_tools/inference_engine/lib/intel64/usb-ma2x8x.mvcmd", + "deployment_tools/inference_engine/lib/intel64/pcie-ma2x8x.mvcmd", + "deployment_tools/inference_engine/lib/intel64/libmyriadPlugin.so", + "deployment_tools/inference_engine/lib/intel64/myriad_compile", + "deployment_tools/inference_engine/lib/intel64/myriad_perfcheck", + "deployment_tools/inference_engine/lib/intel64/vpu_custom_kernels", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_legacy.so", + "install_dependencies/install_NCS_udev_rules.sh" + ] + }, + "gna": { + "ui_name": "Inference Engine Runtime for Intel(R) Gaussian Neural Accelerator", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/external/gna", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_legacy.so", + "deployment_tools/inference_engine/lib/intel64/libGNAPlugin.so" + ] + }, + "hddl": { + "ui_name": "Inference Engine Runtime for Intel(R) Vision Accelerator Design with\n\t Intel(R) Movidius(tm) VPUs", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/lib/intel64/libHDDLPlugin.so", + "deployment_tools/inference_engine/lib/intel64/libinference_engine_legacy.so", + "deployment_tools/inference_engine/external/hddl" + ] + }, + "opencv": { + "ui_name": "OpenCV", + "group": ["opencv"], + "dependencies" : [], + "files": [ + "opencv/version.txt", + "opencv/setupvars.sh", + "opencv/lib", + "opencv/bin" + ] + }, + "python_common": { + "group": ["python"], + "dependencies" : ["ie_core"], + "files": [ + "python/python3", + "python/requirements.txt" + ] + }, + "python3.6": { + "ui_name": "OpenVINO Python API for Python3.6", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.6" + ] + }, + "python3.7": { + "ui_name": "OpenVINO Python API for Python3.7", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.7" + ] + }, + "python3.8": { + "ui_name": "OpenVINO Python API for Python3.8", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.8" + ] + }, + "python3.9": { + "ui_name": "OpenVINO Python API for Python3.9", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.9" + ] + } + } +} diff --git a/scripts/deployment_manager/configs/windows.json b/scripts/deployment_manager/configs/windows.json new file mode 100644 index 00000000000..02fec694d38 --- /dev/null +++ b/scripts/deployment_manager/configs/windows.json @@ -0,0 +1,140 @@ +{ + "version": "0.2", + "components": { + "setupvars": { + "mandatory" : "yes", + "files": [ + "bin" + ] + }, + "openvino_license": { + "mandatory" : "yes", + "files": [ + "licensing" + ] + }, + "ie_core": { + "group": ["ie"], + "files": [ + "deployment_tools/inference_engine/version.txt", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_lp_transformations.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_transformations.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_preproc.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_ir_reader.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_onnx_reader.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_c_api.dll", + "deployment_tools/inference_engine/bin/intel64/Release/AutoPlugin.dll", + "deployment_tools/inference_engine/bin/intel64/Release/plugins.xml", + "deployment_tools/inference_engine/lib/intel64/Release/inference_engine.lib", + "deployment_tools/inference_engine/lib/intel64/Release/inference_engine_transformations.lib", + "deployment_tools/inference_engine/lib/intel64/Release/inference_engine_c_api.lib", + "deployment_tools/inference_engine/external/tbb", + "deployment_tools/ngraph/lib" + ] + }, + "cpu": { + "ui_name": "Inference Engine Runtime for Intel(R) CPU", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/bin/intel64/Release/MKLDNNPlugin.dll" + ] + }, + "gpu": { + "ui_name": "Inference Engine Runtime for Intel(R) Processor Graphics", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/bin/intel64/Release/cache.json", + "deployment_tools/inference_engine/bin/intel64/Release/clDNNPlugin.dll" + ] + }, + "vpu": { + "ui_name": "Inference Engine Runtime for Intel(R) Movidius(tm) VPU", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/bin/intel64/Release/usb-ma2x8x.mvcmd", + "deployment_tools/inference_engine/bin/intel64/Release/pcie-ma2x8x.elf", + "deployment_tools/inference_engine/bin/intel64/Release/myriadPlugin.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_legacy.dll", + "deployment_tools/inference_engine/bin/intel64/Release/myriad_compile.exe", + "deployment_tools/inference_engine/bin/intel64/Release/myriad_perfcheck.exe" + ] + }, + "gna": { + "ui_name": "Inference Engine Runtime for Intel(R) Gaussian Neural Accelerator", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/bin/intel64/Release/gna.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_legacy.dll", + "deployment_tools/inference_engine/bin/intel64/Release/GNAPlugin.dll" + ] + }, + "hddl": { + "ui_name": "Inference Engine Runtime for Intel(R) Vision Accelerator Design with\n\t Intel(R) Movidius(tm) VPUs", + "group": ["ie"], + "dependencies" : ["ie_core"], + "files": [ + "deployment_tools/inference_engine/bin/intel64/Release/HDDLPlugin.dll", + "deployment_tools/inference_engine/bin/intel64/Release/inference_engine_legacy.dll", + "deployment_tools/inference_engine/bin/intel64/Release/hddl_perfcheck.exe", + "deployment_tools/inference_engine/external/MovidiusDriver", + "deployment_tools/inference_engine/external/hddl" + ] + }, + "opencv": { + "ui_name": "OpenCV", + "group": ["opencv"], + "dependencies" : [], + "files": [ + "opencv/version.txt", + "opencv/setupvars.bat", + "opencv/lib", + "opencv/bin" + ] + }, + "python_common": { + "group": ["python"], + "dependencies" : ["ie_core"], + "files": [ + "python/python3", + "python/requirements.txt" + ] + }, + "python3.6": { + "ui_name": "OpenVINO Python API for Python3.6", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.6" + ] + }, + "python3.7": { + "ui_name": "OpenVINO Python API for Python3.7", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.7" + ] + }, + "python3.8": { + "ui_name": "OpenVINO Python API for Python3.8", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.8" + ] + }, + "python3.9": { + "ui_name": "OpenVINO Python API for Python3.9", + "group": ["python"], + "dependencies" : ["ie_core", "python_common"], + "files": [ + "python/python3.9" + ] + } + } +} diff --git a/scripts/deployment_manager/deployman/config.py b/scripts/deployment_manager/deployman/config.py new file mode 100644 index 00000000000..91fff2870a0 --- /dev/null +++ b/scripts/deployment_manager/deployman/config.py @@ -0,0 +1,97 @@ +""" + Copyright (c) 2018-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import os +import platform +from shutil import copytree, copy +import json + + +# class that works with the components from config +class Component: + def __init__(self, name, properties, logger): + self.name = name + for k, v in properties.items(): + setattr(self, k, str(v, 'utf-8') if isinstance(v, bytes) else v) + self.available = True + self.invisible = 'ui_name' not in properties + self.selected = False + self.logger = logger + self.root_dir = os.getenv('INTEL_OPENVINO_DIR', + os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, os.pardir, + os.pardir, os.pardir))) + + def is_exist(self): + self.logger.debug("Checking {} component...".format(self.name)) + for obj in self.files: + obj = os.path.join(self.root_dir, obj) + if not (os.path.isfile(obj) or os.path.isdir(obj)): + self.logger.warning("[{}] Missing {}".format(self.name, obj)) + self.available = False + self.selected = False + return False + return True + + def invert_selection(self): + self.selected = not self.selected + + def set_value(self, attr, value): + setattr(self, attr, value) + + def copy_files(self, destination): + if not self.is_exist(): + raise FileNotFoundError("Files for component {} not found. " + "Please check your OpenVINO installation". + format(self.name)) + else: + if not os.path.exists(destination): + os.makedirs(destination) + for obj in self.files: + src = os.path.join(self.root_dir, obj.strip('\n')) + dst = os.path.join(destination, obj.strip('\n')) + self.logger.debug("[{}] Copy files:: Processing {}...".format(self.name, src)) + if not os.path.exists(os.path.dirname(dst)): + os.makedirs(os.path.dirname(dst)) + if os.path.isdir(src): + copytree(src, dst, symlinks=True) + else: + copy(src, dst) + + +class ComponentFactory: + @staticmethod + def create_component(name, properties, logger): + return Component(name, properties, logger) + + +# class that operating with JSON configs +class ConfigReader: + def __init__(self, logger): + logger.info("Determining the current OS for config selection...") + current_os = platform.system().lower() + cfg_path = os.path.join(os.path.dirname(__file__), os.pardir, + "configs/{}.json".format(current_os)) + if os.path.isfile(cfg_path): + logger.info("Loading {}.cfg...".format(current_os)) + with open(cfg_path, encoding='utf-8') as main_cfg: + self.cfg = json.load(main_cfg) + self.version = self.cfg['version'] + self.components = self.cfg['components'] + logger.info("Successfully loaded.\nConfig version: {}".format(self.version)) + else: + raise RuntimeError("Config can't be found at {}".format(os.path.abspath(cfg_path))) + diff --git a/scripts/deployment_manager/deployman/logger.py b/scripts/deployment_manager/deployman/logger.py new file mode 100644 index 00000000000..4231d7585f1 --- /dev/null +++ b/scripts/deployment_manager/deployman/logger.py @@ -0,0 +1,39 @@ +""" + Copyright (c) 2018-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import logging as log + + +def init_logger(lvl: str): + # create logger + logger = log.getLogger('DeploymentManager') + logger.setLevel(lvl) + + # create console handler and set level to debug + ch = log.StreamHandler() + ch.setLevel(log.DEBUG) + + # create formatter + formatter = log.Formatter('[ %(asctime)s ] %(levelname)s : %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + + return logger + diff --git a/scripts/deployment_manager/deployman/main.py b/scripts/deployment_manager/deployman/main.py new file mode 100644 index 00000000000..c87037244f6 --- /dev/null +++ b/scripts/deployment_manager/deployman/main.py @@ -0,0 +1,219 @@ +""" + Copyright (c) 2018-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +__version__ = '0.6' + +import os +import argparse +import tempfile +from pathlib import Path +from typing import List + +from deployman.logger import init_logger +from deployman.config import ConfigReader, ComponentFactory, Component +from deployman.ui import UserInterface + +logger = init_logger('WARNING') + + +# main class +class DeploymentManager: + def __init__(self, args, selected_targets: List[Component], components: List[Component]): + self.args = args + self.selected_targets = selected_targets + self.components = components + self.dependencies = [] + self.mandatory_components = [] + + def get_dependencies(self): + dependencies_names = [] + logger.debug("Updating dependencies...") + for target in self.selected_targets: + if hasattr(target, 'dependencies'): + dependencies_names.extend(target.dependencies) + # remove duplications + dependencies_names = list(dict.fromkeys(dependencies_names)) + for dependency in dependencies_names: + _target: Component + for _target in self.components: + if _target.name != dependency: + continue + if not _target.is_exist(): + FileNotFoundError("Dependency {} not available.".format(_target.name)) + self.dependencies.append(_target) + + def get_mandatory_component(self): + for _target in self.components: + _target: Component + if hasattr(_target, 'mandatory'): + if not _target.is_exist(): + FileNotFoundError("Mandatory component {} not available.".format(_target.name)) + self.mandatory_components.append(_target) + + @staticmethod + def packing_binaries(archive_name: str, target_dir: str, source_dir: str): + logger.info('Archiving deploy package') + if os.name == 'posix': + archive_path = DeploymentManager.packing_binaries_posix(archive_name, target_dir, source_dir) + else: + archive_path = DeploymentManager.packing_binaries_windows(archive_name, target_dir, source_dir) + logger.setLevel('INFO') + logger.info("Deployment archive is ready. " + "You can find it here:\n\t{}".format(os.path.join(target_dir, archive_path))) + + @staticmethod + def packing_binaries_posix(archive_name: str, target_dir: str, source_dir: str) -> str: + extension = 'tar.gz' + archive_file_name = '{}.{}'.format(archive_name, extension) + archive_path = os.path.join(target_dir, archive_file_name) + + import tarfile + with tarfile.open(archive_path, "w:gz") as tar_pac: + total_files_number = DeploymentManager.count_files_number(source_dir) + count = 0 + logger.info('Total number of files to add to the package: {}'.format(total_files_number)) + for root, dirs, files in os.walk(source_dir): + for file in files: + count += 1 + full_path = os.path.join(root, file) + if not os.path.isfile(full_path): + continue + relative_path = str(Path(full_path).relative_to(source_dir)) + logger.info('Add {} {}/{} file to the package'.format(relative_path, + count, + total_files_number)) + tar_pac.add(full_path, arcname=relative_path) + return archive_path + + @staticmethod + def packing_binaries_windows(archive_name: str, target_dir: str, source_dir: str) -> str: + extension = 'zip' + archive_file_name = '{}.{}'.format(archive_name, extension) + archive_path = os.path.join(target_dir, archive_file_name) + + from zipfile import ZipFile + with ZipFile(archive_path, "w") as zip_pac: + total_files_number = DeploymentManager.count_files_number(source_dir) + count = 0 + logger.info('Total number of files to add to the package: {}'.format(total_files_number)) + for root, dirs, files in os.walk(source_dir): + for file in files: + count += 1 + full_path = os.path.join(root, file) + if not os.path.isfile(full_path): + continue + relative_path = str(Path(full_path).relative_to(source_dir)) + logger.info('Add {} {}/{} file to the package'.format(relative_path, + count, + total_files_number)) + zip_pac.write(os.path.join(root, file), arcname=relative_path) + return archive_path + + @staticmethod + def count_files_number(source_dir: str) -> int: + total_files_number = 0 + for root, dirs, files in os.walk(source_dir): + total_files_number += len(files) + return total_files_number + + def process(self): + # get dependencies if have + self.get_dependencies() + # get mandatory components + self.get_mandatory_component() + + logger.info('Collection information for components') + with tempfile.TemporaryDirectory() as tmpdirname: + for target in self.selected_targets: + target: Component + target.copy_files(tmpdirname) + if self.dependencies: + for dependency in self.dependencies: + dependency: Component + dependency.copy_files(tmpdirname) + if self.mandatory_components: + for target in self.mandatory_components: + target: Component + target.copy_files(tmpdirname) + if self.args.user_data and os.path.exists(self.args.user_data): + from shutil import copytree + logger.info('Storing user data for deploy package ') + copytree(self.args.user_data, + os.path.join( + tmpdirname, + os.path.basename(self.args.user_data.rstrip(os.path.sep))), + symlinks=True) + self.packing_binaries(self.args.archive_name, + self.args.output_dir, tmpdirname) + + +def main(): + # read main config + cfg = ConfigReader(logger) + + # here we store all components + components = [] + + for component in cfg.components: + components.append(ComponentFactory.create_component(component, + cfg.components[component], + logger)) + + # list for only available components + available_targets = [] + help_msg = '' + + for component in components: + if component.is_exist() and not component.invisible: + available_targets.append(component) + help_msg += "{} - {}\n".format(component.name, component.ui_name) + + parser = argparse.ArgumentParser(description="", formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument("--targets", nargs="+", help="List of targets." + "Possible values: \n{}".format(help_msg)) + parser.add_argument("--user_data", type=str, help="Path to user data that will be added to " + "the deployment package", default=None) + parser.add_argument("--output_dir", type=str, help="Output directory for deployment archive", + default=os.getenv("HOME", os.path.join(os.path.join( + os.path.dirname(__file__), os.pardir)))) + parser.add_argument("--archive_name", type=str, help="Name for deployment archive", + default="openvino_deploy_package", ) + parser.add_argument("--version", action="version", version="%(prog)s " + __version__) + + logger.info("Parsing command line arguments") + args = parser.parse_args() + + selected_targets = [] + if not available_targets: + exit("No available targets to packaging detected.\n" + "Please check your OpenVINO installation.") + + ui = UserInterface(__version__, args, available_targets, logger) + if not args.targets: + ui.run() + selected_targets = ui.get_selected_targets() + args = ui.args + else: + for target in args.targets: + target_name = target.lower() + if not any(target_name == _target.name.lower() for _target in available_targets): + raise ValueError("You input incorrect target. {} is not available.".format(target_name)) + for _target in available_targets: + if _target.name.lower() == target_name: + selected_targets.append(_target) + _manager = DeploymentManager(args, selected_targets, components) + _manager.process() diff --git a/scripts/deployment_manager/deployman/ui.py b/scripts/deployment_manager/deployman/ui.py new file mode 100644 index 00000000000..f8831de32ab --- /dev/null +++ b/scripts/deployment_manager/deployman/ui.py @@ -0,0 +1,288 @@ +""" + Copyright (c) 2018-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import os + + +class Button: + def __init__(self, shortcut: str, message: str, action): + self.shortcut = str(shortcut) + self.message = message + self.action = action + + def press(self, *args): + self.action(*args) + + def draw(self): + print("\t{}. {}".format(self.shortcut, self.message)) + + +class ButtonFactory: + @staticmethod + def create_button(shortcut, message, action): + return globals()["Button"](shortcut, message, action) + + +class Controls: + def __init__(self, button_map): + self.controls = [] + for btn in button_map: + self.controls.append( + ButtonFactory.create_button(shortcut=btn["shortcut"], + message=btn["message"], + action=btn["action"])) + + def update_button(self, btn, attr, value): + for _btn in self.controls: + _btn: Button + if btn == _btn.shortcut: + setattr(_btn, attr, value) + + def get_controls(self): + return self.controls + + +# view for User Interface +class ConsoleMenu: + def __init__(self, control_buttons: Controls, user_input_msg: str, hdrs: list, targets=False): + self.control_buttons = control_buttons + self.targets = targets + self.ui_msg = user_input_msg + self.headers = hdrs + + @staticmethod + def clear(): + _ = os.system('clear' if os.name == 'posix' else 'cls') + + @staticmethod + def print_msg(messages): + print("\n".join(messages)) + + @staticmethod + def br(num: int = 1): + print("\n"*num) + + def sync(self, attr, value): + setattr(self, attr, value) + print(getattr(self, attr)) + + def draw(self): + self.clear() + self.print_msg(self.headers) + if self.targets: + for target in self.targets: + print("\t{}. [{}] {}\n".format(self.targets.index(target) + 1, + 'x' if target.selected else ' ', + target.ui_name)) + self.br() + for btn in self.control_buttons.get_controls(): + btn: Button + btn.draw() + return input(self.ui_msg) + + +# class that operating with User Interface +class UserInterface: + def __init__(self, version, args, targets, logger): + self.args = args + self.available_targets = targets + self.is_running = True + self.separator = '-' * 80 + self.user_input = '' + self._active_menu = '' + self._active_controls = '' + self.us_buttons = '' + self.fn_buttons = '' + self.version = version + self.logger = logger + + def get_selected_targets_uinames(self): + return [t.ui_name for t in self.available_targets if t.selected] + + def get_selected_targets(self): + return [t for t in self.available_targets if t.selected] + + @staticmethod + def print_msg(messages): + print("\n".join(messages)) + + def print_selections(self): + for target in self.available_targets: + print("\t{}. [{}] {}\n".format(self.available_targets.index(target) + 1, + 'x' if target.selected else ' ', + target.ui_name)) + + def apply_value_to_targets(self, attr, value): + for target in self.available_targets: + target.set_value(attr, value) + + def select_deselect_all(self): + if any(not target.selected for target in self.available_targets): + self.apply_value_to_targets('selected', True) + else: + self.apply_value_to_targets('selected', False) + + def switch_menu(self, menu: ConsoleMenu, controls): + self._active_menu = menu + self._active_controls = controls + + def process_user_input(self, buttons): + if self.user_input == '': + self.user_input = 'g' + for button in buttons: + if self.user_input == button.shortcut: + button.press() + + def update_output_dir(self): + try: + import readline + readline.parse_and_bind("tab: complete") + readline.set_completer_delims(' \t\n`~!@#$%^&*()-=+[{]}\\|;:\'",<>?') + except ImportError: + # Module readline is not available + pass + self.args.output_dir = input("Please type the full path to the output directory:") + self.fn_buttons: Controls + self.fn_buttons.update_button('o', 'message', "Change output directory [ {} ] ".format( + self.args.output_dir)) + + def update_user_data(self): + try: + import readline + readline.parse_and_bind("tab: complete") + readline.set_completer_delims(' \t\n`~!@#$%^&*()-=+[{]}\\|;:\'",<>?') + except ImportError: + # Module readline is not available + pass + self.args.user_data = input("Please type the full path to the folder with user data:") + self.fn_buttons: Controls + self.fn_buttons.update_button('u', 'message', "Provide(or change) path to folder with user " + "data\n\t (IRs, models, your application," + " and associated dependencies) " + "[ {} ]".format(self.args.user_data)) + + def update_archive_name(self): + self.args.archive_name = input("Please type name of archive without extension:") + self.fn_buttons: Controls + self.fn_buttons.update_button('t', 'message', "Change archive name " + "[ {} ]".format(self.args.archive_name)) + + def dynamic_fn_header_update(self): + return ["Deployment Manager\nVersion " + self.version, + self.separator, "Review the targets below that will be added " + "into the deployment package.\n" + "If needed, change the output directory or " + "add additional user data from the specific folder.\n", + self.separator, "", + "\nSelected targets:\n\t - {}".format( + "\n\t - ".join(self.get_selected_targets_uinames())), "\n" * 2] + + def stop(self): + self.is_running = False + + def run(self): + user_selection_map = [ + { + "shortcut": "a", + "message": "Select/deselect all\n", + "action": self.select_deselect_all, + }, + { + "shortcut": "q", + "message": "Cancel and exit", + "action": exit + } + ] + + finalization_map = [ + { + "shortcut": "b", + "message": "Back to selection dialog", + "action": '', + }, + { + "shortcut": "o", + "message": "Change output directory [ {} ] ".format( + os.path.realpath(self.args.output_dir)), + "action": self.update_output_dir, + }, + { + "shortcut": "u", + "message": "Provide(or change) path to folder with user data\n\t (IRs, models, " + "your application, and associated dependencies) " + "[ {} ]".format(self.args.user_data), + "action": self.update_user_data, + }, + { + "shortcut": "t", + "message": "Change archive name [ {} ]".format(self.args.archive_name), + "action": self.update_archive_name, + }, + { + "shortcut": "g", + "message": "Generate package with current selection [ default ]", + "action": self.stop, + }, + { + "shortcut": "q", + "message": "Cancel and exit", + "action": exit + } + ] + + us_hdrs = ["Deployment Manager\nVersion " + self.version, + self.separator] + self.us_buttons = Controls(user_selection_map) + us_imsg = "\nAdd or remove items by typing the number and hitting \"Enter\"\n" \ + "Press \"Enter\" to continue.\n" + self.separator + "\n" + + fn_hdrs = self.dynamic_fn_header_update() + self.fn_buttons = Controls(finalization_map) + fn_imsg = self.separator + "\nPlease type a selection or press \"Enter\" " + + selection_menu = ConsoleMenu(self.us_buttons, us_imsg, us_hdrs, self.available_targets) + finalization_menu = ConsoleMenu(self.fn_buttons, fn_imsg, fn_hdrs) + + checkboxes = [] + for target in self.available_targets: + checkboxes.append( + ButtonFactory.create_button(shortcut=self.available_targets.index(target) + 1, + message='', + action=target.invert_selection)) + + def switch_fmenu(): + if len(self.get_selected_targets()) > 0: + finalization_menu.sync('headers', self.dynamic_fn_header_update()) + self.switch_menu(finalization_menu, self.fn_buttons.get_controls()) + else: + self.logger.error("Unable to generate package. No components selected.") + switch_usmenu() + + next_btn = Button('g', '', switch_fmenu) + + def switch_usmenu(): + self.switch_menu(selection_menu, + self.us_buttons.get_controls() + checkboxes + [next_btn]) + + self.fn_buttons.update_button('b', 'action', switch_usmenu) + + self._active_menu = selection_menu + self._active_controls = self.us_buttons.get_controls() + checkboxes + [next_btn] + while self.is_running: + self.user_input = self._active_menu.draw().lower() + self.process_user_input(self._active_controls) + return self.available_targets, self.args + diff --git a/scripts/deployment_manager/deployment_manager.py b/scripts/deployment_manager/deployment_manager.py new file mode 100644 index 00000000000..d6cd2413636 --- /dev/null +++ b/scripts/deployment_manager/deployment_manager.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +""" + Copyright (c) 2018-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import sys + +if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 6): + exit("Python* 3.6 or higher is required to run the Deployment Manager.") + +if __name__ == '__main__': + from deployman.main import main + sys.exit(main())