IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Python API] Improve configuration files (#10960)

Browse Source 
* [Python API] Improve configuration files

* fix config files

* update setup.cfd + change quotes

* move all codestyle checks to py_checks job

* update requirements_test.txt

* fix  codestyle according to flake-docstring

* fix

* fix mypy

* apply comments
This commit is contained in:
Anastasia Kuporosova
2022-03-30 20:26:36 +03:00
committed by GitHub
parent be6db5d69a
commit 4c7050f6a9
22 changed files with 378 additions and 1541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
.ci/openvino-onnx/Jenkinsfile vendored
View File

@@ -1,213 +0,0 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
DOCKER_CONTAINER_NAME= "openvino-onnx-ci-container"
DOCKER_IMAGE_TAG = "openvino-onnx-ci-image"
ONNX_MODEL_ZOO_SHA = "d58213534f2a4d1c4b19ba62b3bb5f544353256e"
BACKEND_CONFIGURATIONS = [
[ name: "Release", build_type: "Release" ],
[ name: "Debug", build_type: "Debug" ],
]
// workaround for aborting previous builds on PR update
@NonCPS
def stopPreviousRunningBuilds() {
def jobname = env.JOB_NAME
if (jobname.startsWith("onnx-ci/openvino onnx ci/openvino/PR")){
def buildnum = env.BUILD_NUMBER.toInteger()
def job = Jenkins.instance.getItemByFullName(jobname)
def job_newest = job.builds.first()
for (build in job.builds.reverse()[0..<-1]) {
if (build.isBuilding()){
echo "Stop task = ${build} because newest #${job_newest} is on the way"
build.doStop();
continue;
}
}
}
}
def getGitPrInfo(String project, String workdir) {
def gitPrInfo = [
prAuthorEmail : "",
commitAuthorEmail : "",
commitHash : "",
commitSubject : ""
]
try {
dir ("${workdir}/${project}") {
gitPrInfo.prAuthorEmail = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim()
gitPrInfo.commitAuthorEmail = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim()
gitPrInfo.commitSubject = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim()
gitPrInfo.commitHash = sh (script: 'git log -1 --pretty="format:%s" ', returnStdout: true).trim()
}
}
catch(e) {
echo "Failed to retrieve ${project} git repository information!"
echo "ERROR: ${e}"
}
return gitPrInfo
}
def notifyByEmail(def gitPrInfo) {
stage('Notify') {
String notifyPeople = "${gitPrInfo.prAuthorEmail}, ${gitPrInfo.commitAuthorEmail}"
emailext (
subject: "OpenVino CI: PR ${CHANGE_ID} ${currentBuild.result}!",
body: """
Status: ${currentBuild.result}
Pull Request Title: ${CHANGE_TITLE}
Pull Request: ${CHANGE_URL}
Branch: ${CHANGE_BRANCH}
Commit Hash: ${gitPrInfo.commitSubject}
Commit Subject: ${gitPrInfo.commitHash}
Jenkins Build: ${RUN_DISPLAY_URL}
""",
to: "${notifyPeople}"
)
}
}
def gitSubmoduleUpdate(String repository_name, String workdir) {
dir ("${workdir}/${repository_name}") {
sh label: "Init ${repository_name} submodules",
script:
"""
git submodule init && git submodule update \
--init \
--no-fetch \
--recursive
"""
}
}
def prepare_repository(String workdir) {
dir("${workdir}") {
println "Preparing repository in directory: ${workdir}"
checkout scm
gitSubmoduleUpdate(PROJECT_NAME, workdir)
}
}
def updateModels() {
sh """
./src/bindings/python/tests/test_onnx/model_zoo_preprocess.sh -d ${HOME}/ONNX_CI/models_data -o -s ${ONNX_MODEL_ZOO_SHA}
"""
}
def get_docker_container_name(Map configuration){
println "RUN get_docker_container_name for ${configuration.name}"
String docker_container_name = "${DOCKER_CONTAINER_NAME}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}"
return docker_container_name
}
def buildDockerImage(Map configuration, String workdir) {
String docker_image_tag = "${DOCKER_IMAGE_TAG}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}".toLowerCase()
println "docker_image_tag: ${docker_image_tag}"
updateModels()
sh """
docker build --tag=${docker_image_tag} \
--build-arg BUILD_TYPE=${configuration.build_type} \
--file=.ci/openvino-onnx/Dockerfile \
--build-arg http_proxy=${HTTP_PROXY} \
--build-arg https_proxy=${HTTPS_PROXY} .
"""
}
def runTests(Map configuration, String workdir) {
println "Run tests for ${configuration.name}"
String docker_image_tag = "${DOCKER_IMAGE_TAG}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}".toLowerCase()
String docker_container_name = get_docker_container_name(configuration)
// Run only basic unit tests in Debug configuration
if (configuration.build_type == "Debug") {
sh """
docker run --name ${docker_container_name} ${docker_image_tag}
"""
}
// Run unit-tests AND large model tests by default
else {
sh """
docker run --name ${docker_container_name} \
--volume ${HOME}/ONNX_CI/models_data/model_zoo/onnx_model_zoo_${ONNX_MODEL_ZOO_SHA}:/root/.onnx/model_zoo/onnx_model_zoo \
--volume ${HOME}/ONNX_CI/data/model_zoo/MSFT:/root/.onnx/model_zoo/MSFT \
${docker_image_tag} /bin/bash -c "tox && tox -e zoo_models"
"""
}
}
def getConfigurationsMap() {
def configurationsMap = [:]
for (backend in BACKEND_CONFIGURATIONS) {
def configuration = backend.clone()
configurationsMap[configuration.name] = {
stage(configuration.name) { CONFIGURATION_WORKFLOW(configuration) }
}
}
return configurationsMap
}
CONFIGURATION_WORKFLOW = { configuration ->
node("OpenVINO") {
String workdir = "${HOME}/workspace/${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}"
try {
PROJECT_NAME = "openvino"
stage("Clone repository") {
prepare_repository(workdir)
}
stage("Prepare Docker environment") {
dir("${workdir}") {
buildDockerImage(configuration, workdir)
}
}
stage("Run tests") {
timeout(time: 60, unit: 'MINUTES') {
runTests(configuration, workdir)
}
}
}
catch(e) {
// Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
if ("$e".contains("143")) {
currentBuild.result = "ABORTED"
} else {
currentBuild.result = "FAILURE"
}
def gitPrInfo = getGitPrInfo(PROJECT_NAME, workdir)
notifyByEmail(gitPrInfo)
}
finally {
stage("Cleanup") {
String docker_container_name = get_docker_container_name(configuration)
sh """
docker rm -f ${docker_container_name}
rm -rf ${workdir}
"""
}
}
}
}
pipeline {
agent none
options {
skipDefaultCheckout true
timeout(activity: true, time: 120, unit: 'MINUTES')
}
stages {
stage('Parallel CI') {
steps {
stopPreviousRunningBuilds()
script {
parallelStagesMap = getConfigurationsMap()
parallel parallelStagesMap
}
}
}
}
}

65
.ci/openvino-onnx/watchdog/Jenkinsfile vendored
View File

@@ -1,65 +0,0 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
timeout(30)
{
node(LABEL) {
BUILD_WORKSPACE = "$WORKSPACE/$BUILD_NUMBER"
WATCHDOG_ROOT = "$BUILD_WORKSPACE/.ci/openvino-onnx/watchdog"
VENV_PATH = "${BUILD_WORKSPACE}/.wdvenv"
try {
stage("Clone repository") {
dir ("$BUILD_WORKSPACE") {
checkout([$class: 'GitSCM', branches: [[name: "*/$BRANCH"]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${GITHUB_KEY}", url: "${OPEN_VINO_URL}"]]])
}
}
stage("Prepare environment") {
sh """#!/bin/bash
if [ ! -d ${VENV_PATH} ]; then
python3 -m venv ${VENV_PATH}
source ${VENV_PATH}/bin/activate
pip install -r ${WATCHDOG_ROOT}/requirements.txt
fi
"""
}
stage("Run script") {
withCredentials([
usernamePassword(credentialsId: '7157091e-bc04-42f0-99fd-dc4da2922a55',
usernameVariable: 'username',
passwordVariable: 'password')])
{
dir ("$BUILD_WORKSPACE") {
sh """#!/bin/bash
source ${VENV_PATH}/bin/activate
export PYTHONHTTPSVERIFY=0
python ${WATCHDOG_ROOT}/src/main.py \
--msteams-url=${MSTEAMS_URL_FILE} \
--github-credentials '${username}' '${password}' \
--github-org=${GITHUB_ORG} \
--github-project=${GITHUB_PROJECT} \
--jenkins-token=${JENKINS_TOKEN_FILE} \
--jenkins-server=${JENKINS_SERVER} \
--jenkins-user=${JENKINS_USER} \
--ci-job=${CI_JOB_NAME} \
--watchdog-job=${WATCHDOG_JOB_NAME}
"""
}
}
}
} catch (e) {
echo "$e"
currentBuild.result = "FAILURE"
} finally {
stage("Cleanup") {
sh """
cd $BUILD_WORKSPACE
rm -rf ..?* .[!.]* *
"""
}
}
}
}

6
.ci/openvino-onnx/watchdog/requirements.txt
View File

@@ -1,6 +0,0 @@
python-jenkins==1.7.0
retrying==1.3.3
pygithub==1.51
timeout-decorator==0.4.1
requests==2.23.0
wheel

108
.ci/openvino-onnx/watchdog/src/git_wrapper.py
View File

@@ -1,108 +0,0 @@
#!/usr/bin/python3
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import logging
import timeout_decorator
from datetime import datetime
from retrying import retry
from github import Github, GithubException
# Logging
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
_RETRY_LIMIT = 3
_RETRY_COOLDOWN_MS = 2000
_REQUEST_TIMEOUT_S = 10
class GitWrapper:
"""Class wrapping PyGithub API.
The purpose of this class is to wrap methods from PyGithub API used in Watchdog, for less error-prone and
more convenient use. Docs for used API, including wrapped methods can be found at:
https://pygithub.readthedocs.io/en/latest/introduction.html
:param github_credentials: Credentials used for GitHub
:param repository: GitHub repository name
:param project: GitHub project name
:type github_credentials: String
:type repository: String
:type project: String
"""
def __init__(self, github_credentials, repository, project):
self.git = Github(*github_credentials)
self.repository = repository
self.project = project
self.github_credentials = github_credentials
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_git_time(self):
"""Retrieve time from GitHub.
Used to reliably determine time during Watchdog run.
:return: Datetime object describing current time
:rtype: datetime
"""
try:
datetime_object = self._get_git_time()
except ValueError as e:
raise GitWrapperError(str(e))
except GithubException as e:
message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e))
raise GitWrapperError(message)
except timeout_decorator.TimeoutError:
message = 'GitHub Exception during API status retrieval. Timeout during API request.'
raise GitWrapperError(message)
return datetime_object
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_pull_requests(self):
"""Retrieve paginated list of pull requests from GitHub.
:return: Paginated list of Pull Requests in GitHub repo
:rtype: github.PaginatedList.PaginatedList of github.PullRequest.PullRequest
"""
try:
prs = self._get_pull_requests()
except GithubException as e:
message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e))
raise GitWrapperError(message)
return prs
@timeout_decorator.timeout(_REQUEST_TIMEOUT_S)
def _get_git_time(self):
"""Private method retrieving time from GitHub.
:return: Datetime object describing current time
:rtype: datetime
"""
datetime_string = self.git.get_api_status().raw_headers.get('date', '')
datetime_format = '%a, %d %b %Y %H:%M:%S %Z'
datetime_object = datetime.strptime(datetime_string, datetime_format)
return datetime_object
@timeout_decorator.timeout(_REQUEST_TIMEOUT_S)
def _get_pull_requests(self):
"""Private method retrieving pull requests from GitHub.
:return: Paginated list of Pull Requests in GitHub repo
:rtype: github.PaginatedList.PaginatedList of github.PullRequest.PullRequest
"""
return self.git.get_organization(self.repository).get_repo(self.project).get_pulls()
class GitWrapperError(Exception):
"""Base class for exceptions raised in GitWrapper.
:param message Explanation of the error
"""
def __init__(self, message):
self.message = message
log.exception(message)

91
.ci/openvino-onnx/watchdog/src/jenkins_wrapper.py
View File

@@ -1,91 +0,0 @@
#!/usr/bin/python3
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import requests
import jenkins
import logging
from retrying import retry
# Logging
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
_RETRY_LIMIT = 3
_RETRY_COOLDOWN_MS = 5000
class JenkinsWrapper:
"""Class wrapping Python-Jenkins API.
The purpose of this class is to wrap methods from Python-Jenkins API used in Watchdog, for less error-prone and
more convenient use. Docs for used API, including wrapped methods can be found at:
https://python-jenkins.readthedocs.io/en/latest/
:param jenkins_token: Token used for Jenkins
:param jenkins_user: Username used to connect to Jenkins
:param jenkins_server: Jenkins server address
:type jenkins_token: String
:type jenkins_user: String
:type jenkins_server: String
"""
def __init__(self, jenkins_token, jenkins_user, jenkins_server):
self.jenkins_server = jenkins_server
self.jenkins = jenkins.Jenkins(jenkins_server, username=jenkins_user,
password=jenkins_token)
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_build_console_output(self, job_name, build_number):
return self.jenkins.get_build_console_output(job_name, build_number)
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_job_info(self, job_name):
return self.jenkins.get_job_info(job_name)
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_build_info(self, job_name, build_number):
return self.jenkins.get_build_info(job_name, build_number)
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_queue_item(self, queue_id):
"""Attempt to retrieve Jenkins job queue item.
Exception communicating queue doesn't exist is expected,
in that case method returns empty dict.
:param queue_id: Jenkins job queue ID number
:type queue_id: int
:return: Dictionary representing Jenkins job queue item
:rtype: dict
"""
try:
return self.jenkins.get_queue_item(queue_id)
except Exception as e:
# Exception 'queue does not exist' is expected behaviour when job is running
if 'queue' in str(e) and 'does not exist' in str(e):
return {}
else:
raise
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_idle_ci_hosts(self):
"""Query Jenkins for idle servers.
Send GET request to Jenkins server, querying for idle servers labeled
for OpenVino-ONNX CI job.
:return: Number of idle hosts delegated to OpenVino-ONNX CI
:rtype: int
"""
jenkins_request_url = self.jenkins_server + 'label/ci&&onnx/api/json?pretty=true'
try:
log.info('Sending request to Jenkins: %s', jenkins_request_url)
r = requests.Request(method='GET', url=jenkins_request_url, verify=False)
response = self.jenkins.jenkins_request(r).json()
return int(response['totalExecutors']) - int(response['busyExecutors'])
except Exception as e:
log.exception('Failed to send request to Jenkins!\nException message: %s', str(e))
raise

89
.ci/openvino-onnx/watchdog/src/main.py
View File

@@ -1,89 +0,0 @@
#!/usr/bin/python3
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import argparse
import sys
from watchdog import Watchdog
DEFAULT_MSTEAMS_URL_FILE = '/home/lab_nerval/tokens/msteams_url'
DEFAULT_GITHUB_ORGANIZATION = 'openvinotoolkit'
DEFAULT_GITHUB_PROJECT = 'openvino'
DEFAULT_JENKINS_TOKEN_FILE = '/home/lab_nerval/tokens/crackerjack'
DEFAULT_JENKINS_SERVER = 'https://crackerjack.intel.com/'
DEFAULT_JENKINS_USER = 'lab_nerval'
DEFAULT_CI_JOB_NAME = 'onnx/OpenVino_CI'
DEFAULT_WATCHDOG_JOB_NAME = 'onnx/ci_watchdog'
def main(args):
"""
Read args passed to script, load tokens and run watchdog.
Keyword arguments:
:param args: arguments parsed by argparse ArgumentParser
:return: returns status code 0 on successful completion
"""
jenkins_server = args.jenkins_server.strip()
jenkins_user = args.jenkins_user.strip()
jenkins_token = open(args.jenkins_token).read().replace('\n', '').strip()
msteams_url = open(args.msteams_url).read().replace('\n', '').strip()
github_credentials = args.github_credentials
github_org = args.github_org
github_project = args.github_project
ci_job = args.ci_job.strip()
watchdog_job = args.watchdog_job.strip()
quiet = args.quiet
wd = Watchdog(jenkins_token=jenkins_token,
jenkins_server=jenkins_server,
jenkins_user=jenkins_user,
github_credentials=github_credentials,
git_org=github_org,
git_project=github_project,
msteams_url=msteams_url,
ci_job_name=ci_job,
watchdog_job_name=watchdog_job)
wd.run(quiet=quiet)
return 0
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--msteams-url', help='Path to MS Teams channel url to communicate messages.',
default=DEFAULT_MSTEAMS_URL_FILE, action='store', required=False)
parser.add_argument('--github-credentials', help='GitHub user credentials to access repo.',
nargs="+", required=True)
parser.add_argument('--github-org', help='Name of organization on GitHub.',
default=DEFAULT_GITHUB_ORGANIZATION, action='store', required=False)
parser.add_argument('--github-project', help='Name of project on GitHub.',
default=DEFAULT_GITHUB_PROJECT, action='store', required=False)
parser.add_argument('--jenkins-token', help='Path to Jenkins user token to access build info.',
default=DEFAULT_JENKINS_TOKEN_FILE, action='store', required=False)
parser.add_argument('--jenkins-server', help='Jenkins server address.',
default=DEFAULT_JENKINS_SERVER, action='store', required=False)
parser.add_argument('--jenkins-user', help='Jenkins user used to log in.',
default=DEFAULT_JENKINS_USER, action='store', required=False)
parser.add_argument('--ci-job', help='Jenkins CI job name.',
default=DEFAULT_CI_JOB_NAME, action='store', required=False)
parser.add_argument('--watchdog-job', help='Jenkins CI Watchdog job name.',
default=DEFAULT_WATCHDOG_JOB_NAME, action='store', required=False)
parser.add_argument('--quiet', help="Quiet mode - doesn\'t send message to communicator.",
action='store_true', required=False)
args = parser.parse_args()
sys.exit(main(args))

128
.ci/openvino-onnx/watchdog/src/ms_teams_communicator.py
View File

@@ -1,128 +0,0 @@
#!/usr/bin/python3
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import requests
class MSTeamsCommunicator:
"""Class communicating with MSTeams using Incoming Webhook.
The purpose of this class is to use MSTeams API to send message.
Docs for used API, including wrapped methods can be found at:
https://docs.microsoft.com/en-us/outlook/actionable-messages/send-via-connectors
"""
def __init__(self, _ci_alerts_channel_url):
self._ci_alerts_channel_url = _ci_alerts_channel_url
self._queued_messages = {
self._ci_alerts_channel_url: [],
}
@property
def messages(self):
"""
Get list of queued messages.
:return: List of queued messages
:return type: List[String]
"""
return self._queued_messages.values()
def queue_message(self, message):
"""
Queue message to be sent later.
:param message: Message content
:type message: String
"""
self._queued_messages[self._ci_alerts_channel_url].append(message)
def _parse_text(self, watchdog_log, message):
"""
Parse text to display as alert.
:param watchdog_log: Watchdog log content
:param message: Unparsed message content
:type watchdog_log: String
:type message: String
"""
message_split = message.split('\n')
log_url = None
if len(message_split) == 3:
log_url = message_split[-1]
title = message_split[0]
text = message_split[1]
header = watchdog_log.split(' - ')
header_formatted = '{} - [Watchdog Log]({})'.format(header[0], header[1])
return title, log_url, '{}\n\n{}'.format(header_formatted, text)
def _json_request_content(self, title, log_url, text_formatted):
"""
Create final json request to send message to MS Teams channel.
:param title: Title of alert
:param log_url: URL to PR
:param text_formatted: General content of alert - finally formatted
:type title: String
:type title: String
:type title: String
"""
data = {
'@context': 'https://schema.org/extensions',
'@type': 'MessageCard',
'themeColor': '0072C6',
'title': title,
'text': text_formatted,
'potentialAction':
[
{
'@type': 'OpenUri',
'name': 'Open PR',
'targets':
[
{
'os': 'default',
'uri': log_url,
},
],
},
],
}
return data
def _send_to_channel(self, watchdog_log, message_queue, channel_url):
"""
Send MSTeams message to specified channel.
:param watchdog_log: Watchdog log content
:param message_queue: Queued messages to send
:param channel_url: Channel url
:type watchdog_log: String
:type message_queue: String
:type channel_url: String
"""
for message in message_queue:
title, log_url, text_formatted = self._parse_text(watchdog_log, message)
data = self._json_request_content(title, log_url, text_formatted)
try:
requests.post(url=channel_url, json=data)
except Exception as ex:
raise Exception('!!CRITICAL!! MSTeamsCommunicator: Could not send message '
'due to {}'.format(ex))
def send_message(self, watchdog_log, quiet=False):
"""
Send queued messages as single communication.
:param watchdog_log: Watchdog log content
:param quiet: Flag for disabling sending report through MS Teams
:type watchdog_log: String
:type quiet: Boolean
"""
for channel, message_queue in self._queued_messages.items():
if not quiet and message_queue:
self._send_to_channel(watchdog_log, message_queue, channel)

505
.ci/openvino-onnx/watchdog/src/watchdog.py
View File

@@ -1,505 +0,0 @@
#!/usr/bin/python3
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import datetime
import time
import re
import logging
import requests
from ms_teams_communicator import MSTeamsCommunicator
from jenkins_wrapper import JenkinsWrapper
from jenkins import NotFoundException
from git_wrapper import GitWrapper, GitWrapperError
import os
import json
# Logging
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
# Watchdog static constant variables
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
_BUILD_DURATION_THRESHOLD = datetime.timedelta(minutes=60)
_CI_START_THRESHOLD = datetime.timedelta(minutes=30)
_AWAITING_JENKINS_THRESHOLD = datetime.timedelta(minutes=5)
_WATCHDOG_DIR = os.path.expanduser('~')
_PR_REPORTS_CONFIG_KEY = 'pr_reports'
_CI_BUILD_FAIL_MESSAGE = 'ERROR: py3: commands failed'
_CI_BUILD_SUCCESS_MESSAGE = 'py3: commands succeeded'
_GITHUB_CI_CHECK_NAME = 'OpenVINO-ONNX'
INTERNAL_ERROR_MESSAGE_HEADER = '!!! --- !!! INTERNAL WATCHDOG ERROR !!! --- !!!'
ERROR_MESSAGE_HEADER = '!!! OpenVino-ONNX CI Error !!!'
WARNING_MESSAGE_HEADER = 'OpenVino-ONNX CI WARNING'
INFO_MESSAGE_HEADER = 'OpenVino-ONNX CI INFO'
class Watchdog:
"""Class describing OpenVino-ONNX-CI Watchdog.
Watchdog connects to GitHub and retrieves the list of current pull requests (PRs) in
OpenVino repository. Then it connects to specified Jenkins server to
check CI jobs associated with every PR. Watchdog verifies time durations for Jenkins
initial response, job queue and execution against time treshold constants. Every fail
is logged and reported through MS Teams communicators.
:param jenkins_token: Token used for Jenkins
:param jenkins_server: Jenkins server address
:param jenkins_user: Username used to connect to Jenkins
:param github_credentials: Credentials used to connect to GitHub
:param msteams_url: URL used to connect to MS Teams channel
:param ci_job_name: OpenVino-ONNX CI job name used in Jenkins
:param watchdog_job_name: Watchdog job name used in Jenkins
:type jenkins_token: String
:type jenkins_server: String
:type jenkins_user: String
:type github_credentials: String
:type msteams_url: String
:type ci_job_name: String
:type watchdog_job_name: String
.. note::
Watchdog and OpenVino-ONNX CI job must be placed on the same Jenkins server.
"""
def __init__(self, jenkins_token, jenkins_server, jenkins_user, github_credentials, git_org,
git_project, msteams_url, ci_job_name, watchdog_job_name):
self._config_path = os.path.join(_WATCHDOG_DIR, '{}/.{}_ci_watchdog.json'.format(_WATCHDOG_DIR, git_project))
# Jenkins Wrapper object for CI job
self._jenkins = JenkinsWrapper(jenkins_token,
jenkins_user=jenkins_user,
jenkins_server=jenkins_server)
# Load GitHub token and log in, retrieve pull requests
self._git = GitWrapper(github_credentials, repository=git_org, project=git_project)
# Create MS Teams api object
self._msteams_hook = MSTeamsCommunicator(msteams_url)
self._ci_job_name = ci_job_name.lower()
self._watchdog_job_name = watchdog_job_name
# Read config file
self._config = self._read_config_file()
# Time at Watchdog initiation
self._now_time = datetime.datetime.now()
self._current_prs = {}
self._ms_teams_enabled = True
def run(self, quiet=False):
"""Run main watchdog logic.
Retrieve list of pull requests and pass it to the method responsible for checking them.
:param quiet: Flag for disabling sending report through communicator
:type quiet: Boolean
"""
try:
pull_requests = self._git.get_pull_requests()
except GitWrapperError:
message = 'Failed to retrieve Pull Requests!'
log.exception(message)
self._queue_message(message, message_severity='internal')
# Check all pull requests
for pr in pull_requests:
try:
self._check_pr(pr)
except Exception as e:
log.exception(str(e))
self._queue_message(str(e), message_severity='internal', pr=pr)
self._update_config()
self._send_message(quiet=quiet)
def _read_config_file(self):
"""Read Watchdog config file stored on the system.
The file stores every fail already reported along with timestamp. This
mechanism is used to prevent Watchdog from reporting same failure
multiple times. In case there's no config under the expected path,
appropriate data structure is created and returned.
:return: Returns dict of dicts with reported fails with their timestamps
:rtype: dict of dicts
"""
if os.path.isfile(self._config_path):
log.info('Reading config file in: {}'.format(self._config_path))
file = open(self._config_path, 'r')
data = json.load(file)
else:
log.info('No config file found in: {}'.format(self._config_path))
data = {_PR_REPORTS_CONFIG_KEY: {}}
return data
def _check_pr(self, pr):
"""Check pull request (if there's no reason to skip).
Retrieve list of statuses for every PR's last commit and interpret them. Filters out statuses
unrelated to OpenVino-ONNX Jenkins CI and passes relevant statuses to method that interprets them.
If no commit statuses related to Jenkins are available after time defined by
**_AWAITING_JENKINS_THRESHOLD** calls appropriate method to check for builds waiting in queue.
:param pr: GitHub Pull Requests
:type pr: github.PullRequest.PullRequest
"""
log.info('===============================================')
log.info('Checking PR#{}'.format(pr.number))
# Get last Jenkins status
last_status = self._get_last_status(pr)
# Append PR checked in current run for Watchdog config
self._current_prs[str(pr.number)] = self._get_pr_timestamps(pr, last_status)
if self._should_ignore(pr) or self._updated_since_last_run(pr):
log.info('Ignoring PR#{}'.format(pr.number))
return
# Calculate time passed since PR update (any commit, merge or comment)
pr_time_delta = self._now_time - pr.updated_at
if last_status:
# Interpret found CI statuses
log.info('Last status: {} at {}'.format(last_status.description, last_status.updated_at))
self._interpret_status(last_status, pr)
elif pr_time_delta > _CI_START_THRESHOLD:
# If there's no status after assumed time - check if build is waiting in queue
log.info('CI for PR {}: NO JENKINS STATUS YET'.format(pr.number))
self._check_missing_status(pr)
@staticmethod
def _get_pr_timestamps(pr, last_status):
"""Get dict containing PR timestamp and last status timestamp.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
:return: Dictionary with PR and last status update timestamps
:rtype: dict
"""
pr_timestamp = time.mktime(pr.updated_at.timetuple())
if last_status:
status_timestamp = time.mktime(last_status.updated_at.timetuple())
else:
status_timestamp = None
pr_dict = {'pr_timestamp': pr_timestamp,
'status_timestamp': status_timestamp}
return pr_dict
@staticmethod
def _get_last_status(pr):
"""Get last commit status posted from Jenkins.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
:return: Either last PR status posted from Jenkins or None
:rtype: github.CommitStatus.CommitStatus
"""
# Find last commit in PR
last_commit = pr.get_commits().reversed[0]
# Get statuses and filter them to contain only those related to Jenkins CI
# and check if CI in Jenkins started
statuses = last_commit.get_statuses()
jenk_statuses = [stat for stat in statuses if
_GITHUB_CI_CHECK_NAME in stat.context]
try:
last_status = jenk_statuses[0]
except IndexError:
last_status = None
return last_status
@staticmethod
def _should_ignore(pr):
"""Determine if PR should be ignored.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
:return: Returns True if PR should be ignored
:rtype: Bool
"""
# Ignore PR if it has WIP label or WIP in title
if 'WIP' in pr.title:
log.info('PR#{} should be ignored. WIP tag in title.'.format(pr.number))
return True
label_names = [label.name for label in pr.labels]
if 'WIP' in label_names:
log.info('PR#{} should be ignored. WIP label present.'.format(pr.number))
return True
# Ignore PR if base ref is not master
if 'master' not in pr.base.ref:
log.info('PR#{} should be ignored. Base ref is not master'.format(pr.number))
return True
# Ignore PR if mergeable state is 'dirty' or 'behind'.
# Practically this ignores PR in case of merge conflicts
ignored_mergeable_states = ['behind', 'dirty', 'draft']
if pr.mergeable_state in ignored_mergeable_states:
log.info('PR#{} should be ignored. Mergeable state is {}. '.format(pr.number, pr.mergeable_state))
return True
# If no criteria for ignoring PR are met - return false
return False
def _updated_since_last_run(self, pr):
# Ignore if PR was already checked and there was no update in meantime
pr_number = str(pr.number)
current_pr_timestamps = self._current_prs.get(pr_number)
last_pr_timestamps = self._config[_PR_REPORTS_CONFIG_KEY].get(pr_number)
if current_pr_timestamps == last_pr_timestamps:
log.info('PR#{} - No update since last check'.format(pr.number))
return True
else:
return False
def _check_missing_status(self, pr):
"""Verify if missing status is expected.
This method checks if CI build for last was scheduled and still waits in queue for
executor.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
"""
pr_time_delta = self._now_time - pr.updated_at
try:
build_number = self._build_scheduled(pr)
if self._build_in_queue(pr, build_number):
message = ('PR# {}: build waiting in queue after {} minutes.'
.format(pr.number, pr_time_delta.seconds / 60))
severity = 'warning'
else:
message = ('PR# {}: missing status on GitHub after {} minutes.'
.format(pr.number, pr_time_delta.seconds / 60))
severity = 'error'
self._queue_message(message, message_severity=severity, pr=pr)
except TypeError:
log.info('Committer outside of OpenVino organization')
def _build_scheduled(self, pr):
"""Check if Jenkins build corresponding to PR was scheduled.
This method takes last Jenkins build for given PR and compares hash from Jenkins console output
and sha from PR object to determine if CI build for appropriate commit was scheduled.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
:return: Returns build number or -1 if no build found
:rtype: int
"""
pr_number = str(pr.number)
project_name_full = self._ci_job_name + '/PR-' + pr_number
try:
# Retrieve console output from last Jenkins build for job corresponding to this PR
last_build_number = self._jenkins.get_job_info(project_name_full)['lastBuild']['number']
console_output = self._jenkins.get_build_console_output(project_name_full, last_build_number)
# Check if CI build was scheduled - commit hash on GH must match hash in last Jenkins build console output
# Retrieve hash from Jenkins output
match_string = '(?:Obtained .ci/[a-zA-Z/]+Jenkinsfile from ([a-z0-9]{40}))'
retrieved_sha = re.search(match_string, console_output).group(1)
if retrieved_sha == pr.get_commits().reversed[0].sha:
return last_build_number
else:
return -1
except (NotFoundException, AttributeError, requests.exceptions.HTTPError):
message = ('PR #{}: Jenkins build corresponding to commit {} not found!'
.format(pr_number, pr.get_commits().reversed[0].sha))
self._queue_message(message, message_severity='error', pr=pr)
return -1
def _build_in_queue(self, pr, build_number):
"""Check if Jenkins build waits in queue.
This method verifies if CI build is waiting in queue based on console output.
:param pr: Single PR being currently checked
:param build_number: Jenkins build number to retrieve console output from
:type pr: github.PullRequest.PullRequest
:type build_number: int
:return: Returns True if CI build is waiting in queue
:rtype: Bool
"""
pr_number = str(pr.number)
project_name_full = self._ci_job_name + '/PR-' + pr_number
# Retrieve console output
try:
console_output = self._jenkins.get_build_console_output(project_name_full, build_number)
except NotFoundException:
return False
# Check if build is waiting in queue (and not already running on an executor)
if 'Waiting for next available executor on' in console_output \
and 'Running on' not in console_output:
log.info('CI for PR %s: WAITING IN QUEUE', pr_number)
return True
else:
return False
def _interpret_status(self, status, pr):
"""
Verify GitHub status passed to the method.
This method verifies last commit status for given PR, calling appropriate methods
to further validate the status.
:param status: GitHub commit status
:param pr: Single PR being currently checked
:type status: github.CommitStatus.CommitStatus
:type pr: github.PullRequest.PullRequest
"""
try:
# Retrieve build number for Jenkins build related to this PR
build_number = self._retrieve_build_number(status.target_url)
# CI build finished - verify if expected output is present
finished_statuses = ['Build finished', 'This commit cannot be built', 'This commit looks good']
pending_statuses = ['This commit is being built', 'Testing in progress',
'This commit is scheduled to be built']
if any(phrase in status.description for phrase in finished_statuses):
self._check_finished(pr, build_number)
# CI build in progress - verify timeouts for build queue and duration
elif any(phrase in status.description for phrase in pending_statuses):
self._check_in_progress(pr, build_number)
else:
message = 'ONNX CI job for PR# {}: unrecognized status: {}'.format(pr.number, status.description)
self._queue_message(message, message_severity='error', pr=pr)
except Exception:
# Log Watchdog internal error in case any status can't be properly verified
message = 'Failed to verify status "{}" for PR# {}'.format(status.description, pr.number)
log.exception(message)
self._queue_message(message, message_severity='internal', pr=pr)
def _retrieve_build_number(self, url):
"""Retrieve Jenkins CI job build number from URL address coming from GitHub commit status.
:param url: URL address from GitHub commit status
:type url: String
:return: Returns build number
:rtype: int
"""
# Retrieve the build number from url string
match_obj = re.search('(?:/PR-[0-9]+/)([0-9]+)', url)
try:
number = int(match_obj.group(1))
return number
except Exception:
log.exception('Failed to retrieve build number from url link: %s', url)
raise
def _queue_message(self, message, message_severity='info', pr=None):
"""Add a message to message queue in communicator object.
The queued message is constructed based on message string passed as
a method argument and message header. Message header is mapped to message severity
also passed as an argument.
:param message: Message content
:param message_severity: Message severity level
:type message: String
:type message_severity: int
"""
log.info(message)
internal = False
if 'internal' in message_severity:
message_header = INTERNAL_ERROR_MESSAGE_HEADER
internal = True
elif 'error' in message_severity:
message_header = ERROR_MESSAGE_HEADER
elif 'warning' in message_severity:
message_header = WARNING_MESSAGE_HEADER
else:
message_header = INFO_MESSAGE_HEADER
# If message is related to PR attatch url
if pr:
message = message + '\n' + pr.html_url
send = message_header + '\n' + message
if self._ms_teams_enabled:
self._msteams_hook.queue_message(send)
def _check_finished(self, pr, build_number):
"""Verify if finished build output contains expected string for either fail or success.
:param pr: Single PR being currently checked
:param build_number: Jenkins CI job build number
:type pr: github.PullRequest.PullRequest
:type build_number: int
"""
pr_number = str(pr.number)
log.info('CI for PR %s: FINISHED', pr_number)
# Check if FINISH was valid FAIL / SUCCESS
project_name_full = self._ci_job_name + '/PR-' + pr_number
build_output = self._jenkins.get_build_console_output(project_name_full, build_number)
if _CI_BUILD_FAIL_MESSAGE not in build_output \
and _CI_BUILD_SUCCESS_MESSAGE not in build_output:
message = ('ONNX CI job for PR #{}: finished but no tests success or fail '
'confirmation is present in console output!'.format(pr_number))
self._queue_message(message, message_severity='error', pr=pr)
def _send_message(self, quiet=False):
"""Send messages queued in MS Teams objects to designated channel.
Queued messages are being sent as a single communication.
:param quiet: Flag for disabling sending report through communicator
:type quiet: Boolean
"""
if any(messages for messages in self._msteams_hook.messages):
try:
watchdog_build = self._jenkins.get_job_info(self._watchdog_job_name)['lastBuild']
watchdog_build_number = watchdog_build['number']
watchdog_build_link = watchdog_build['url']
except Exception:
watchdog_build_number = 'UNKNOWN'
watchdog_build_link = self._jenkins.jenkins_server
send = self._watchdog_job_name + '- build ' + str(
watchdog_build_number) + ' - ' + watchdog_build_link
if self._ms_teams_enabled:
self._msteams_hook.send_message(send, quiet=quiet)
else:
log.info('Nothing to report.')
def _check_in_progress(self, pr, build_number):
"""Check if CI build succesfully started.
Checks if build started within designated time threshold, and job is
currently running - it didn't cross the time threshold.
:param pr: Single PR being currently checked
:param build_number: Jenkins CI job build number
:type pr: github.PullRequest.PullRequest
:type build_number: int
"""
pr_number = str(pr.number)
log.info('CI for PR %s: TESTING IN PROGRESS', pr_number)
project_name_full = self._ci_job_name + '/PR-' + pr_number
build_info = self._jenkins.get_build_info(project_name_full, build_number)
build_datetime = datetime.datetime.fromtimestamp(build_info['timestamp'] / 1000.0)
build_delta = self._now_time - build_datetime
log.info('Build %s: IN PROGRESS, started: %s minutes ago', str(build_number),
str(build_delta))
# If build still waiting in queue
if build_delta > _CI_START_THRESHOLD and self._build_in_queue(pr, build_number):
message = ('ONNX CI job build #{}, for PR #{} waiting in queue after {} '
'minutes'.format(build_number, pr_number, str(build_delta.seconds / 60)))
self._queue_message(message, message_severity='warning', pr=pr)
elif build_delta > _BUILD_DURATION_THRESHOLD:
# CI job take too long, possibly froze - communicate failure
message = ('ONNX CI job build #{}, for PR #{} started, '
'but did not finish in designated time of {} '
'minutes!'.format(build_number, pr_number,
str(_BUILD_DURATION_THRESHOLD.seconds / 60)))
self._queue_message(message, message_severity='error', pr=pr)
def _update_config(self):
"""Update Watchdog config file with PRs checked in current Watchdog run, remove old entries.
:param current_prs: List of PR numbers checked during current Watchdog run
:type current_prs: list of ints
"""
# Cleanup config of old reports
log.info('Writing to config file at: {}'.format(self._config_path))
new_config = {_PR_REPORTS_CONFIG_KEY: self._current_prs}
file = open(self._config_path, 'w+')
json.dump(new_config, file)

69
.github/workflows/py_checks.yml vendored
View File

@@ -1,4 +1,4 @@
name: IE Python Checks
name: Python API Checks
on:
workflow_dispatch:
@@ -23,8 +23,9 @@ jobs:
with:
python-version: '3.6'
- name: Install dependencies
run: python -m pip install -r src/bindings/python/src/compatibility/openvino/requirements_dev.txt
- name: Run Flake on samples
run: python -m pip install -r src/bindings/python/requirements_test.txt
# samples code-style
- name: Run flake8 on samples
run: python -m flake8 ./ --config=setup.cfg
working-directory: samples/python
- name: Create code style diff for samples
@@ -38,21 +39,53 @@ jobs:
with:
name: samples_diff
path: samples_diff.diff
- name: Run Flake on src
# IE Python API Flake code-style
- name: Run flake8 on IE Python API
run: python -m flake8 ./ --config=setup.cfg
working-directory: src/bindings/python/src/compatibility/openvino
- name: Create code style diff for Python src
- name: Create code style diff for IE Python API
if: failure()
run: |
python -m black -l 160 -S ./
git diff > src_diff.diff
git diff > ie_python_diff.diff
working-directory: src/bindings/python/src/compatibility/openvino
- uses: actions/upload-artifact@v2
if: failure()
with:
name: src_diff
path: src_diff.diff
- name: Run Flake on wheel
name: ie_python_diff
path: ie_python_diff.diff
# nGraph Python API Flake code-style
- name: Run flake8 on nGraph Python API
run: python -m flake8 ./src/compatibility/ngraph --config=setup.cfg
working-directory: src/bindings/python
- name: Create code style diff for nGraph Python API
if: failure()
run: |
python -m black -l 160 -S ./
git diff > pyngraph_diff.diff
working-directory: src/bindings/python/src/compatibility/ngraph
- uses: actions/upload-artifact@v2
if: failure()
with:
name: pyngraph_diff
path: pyngraph_diff.diff
# Python API 2.0 Flake code-style
- name: Run flake8 on Python API 2.0
run: python -m flake8 ./src/openvino --config=setup.cfg
working-directory: src/bindings/python
- name: Create code style diff for Python API 2.0
if: failure()
run: |
python -m black -l 160 -S ./
git diff > pyopenvino_diff.diff
working-directory: src/bindings/python/src/openvino
- uses: actions/upload-artifact@v2
if: failure()
with:
name: pyopenvino_diff
path: pyopenvino_diff.diff
# wheel Flake code-style
- name: Run flake8 on wheel
run: python -m flake8 ./ --config=../setup.cfg
working-directory: src/bindings/python/wheel
- name: Create code style diff for wheel
@@ -66,10 +99,24 @@ jobs:
with:
name: wheel_diff
path: wheel_diff.diff
- name: Run MyPy
# Python API 2.0 tests Flake code-style
- name: Run flake8 on python tests
# ignore lack of docs in tests
run: python -m flake8 tests/ --config=setup.cfg
working-directory: src/bindings/python
# IE Python API mypy check
- name: Run mypy on IE Python API
run: python -m mypy ./ --config-file ./setup.cfg
working-directory: src/bindings/python/src/compatibility/openvino
# nGraph Python API mypy check
- name: Run mypy on nGraph Python API
run: python -m mypy ./src/compatibility/ngraph --config-file ./setup.cfg
working-directory: src/bindings/python
# Python API 2.0 mypy check
- name: Run mypy on Python API 2.0
run: python -m mypy ./src/openvino --config-file ./setup.cfg
working-directory: src/bindings/python
- name: Run Bandit
run: python -m bandit -r ./ -f screen
working-directory: src/bindings/python/src/compatibility/openvino

2
samples/python/classification_sample_async/classification_sample_async.py
View File

@@ -14,7 +14,7 @@ from openvino.runtime import AsyncInferQueue, Core, InferRequest, Layout, Type
def parse_args() -> argparse.Namespace:
"""Parse and return command line arguments"""
"""Parse and return command line arguments."""
parser = argparse.ArgumentParser(add_help=False)
args = parser.add_argument_group('Options')
# fmt: off

2
samples/python/hello_query_device/hello_query_device.py
View File

@@ -9,7 +9,7 @@ from openvino.runtime import Core
def param_to_string(parameters) -> str:
"""Convert a list / tuple of parameters returned from IE to a string"""
"""Convert a list / tuple of parameters returned from IE to a string."""
if isinstance(parameters, (list, tuple)):
return ', '.join([str(x) for x in parameters])
else:

2
samples/python/model_creation_sample/model_creation_sample.py
View File

@@ -16,7 +16,7 @@ from data import digits
def create_ngraph_function(model_path: str) -> Model:
"""Create a model on the fly from the source code using ngraph"""
"""Create a model on the fly from the source code using ngraph."""
def shape_and_length(shape: list) -> typing.Tuple[list, int]:
length = reduce(lambda x, y: x * y, shape)

6
samples/python/setup.cfg
View File

@@ -1,7 +1,11 @@
[flake8]
# ignore:
# D100 - Missing docstring in public module
# D101 - Missing docstring in public class
# D103 - Missing docstring in public function
filename = *.py
max-line-length = 160
ignore = E203
ignore = E203,D100,D101,D103
max-parameters-amount = 8
show_source = True
docstring-convention = google

4
samples/python/speech_sample/arg_parser.py
View File

@@ -5,7 +5,7 @@ import argparse
def build_arg_parser() -> argparse.ArgumentParser:
"""Create and return argument parser"""
"""Create and return argument parser."""
parser = argparse.ArgumentParser(add_help=False)
args = parser.add_argument_group('Options')
model = parser.add_mutually_exclusive_group(required=True)
@@ -73,7 +73,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
def parse_args() -> argparse.Namespace:
"""Parse and validate command-line arguments"""
"""Parse and validate command-line arguments."""
parser = build_arg_parser()
args = parser.parse_args()

12
samples/python/speech_sample/file_options.py
View File

@@ -15,9 +15,9 @@ class FileData(NamedTuple):
def read_ark_file(file_name: str) -> FileData:
"""Read utterance matrices from a .ark file"""
"""Read utterance matrices from a .ark file."""
def read_key(input_file: IO[Any]) -> str:
"""Read a identifier of utterance matrix"""
"""Read a identifier of utterance matrix."""
key = ''
char = input_file.read(1).decode()
@@ -28,7 +28,7 @@ def read_ark_file(file_name: str) -> FileData:
return key
def read_matrix(input_file: IO[Any]) -> np.ndarray:
"""Read a utterance matrix"""
"""Read a utterance matrix."""
header = input_file.read(5).decode()
if 'FM' in header:
num_of_bytes = 4
@@ -61,7 +61,7 @@ def read_ark_file(file_name: str) -> FileData:
def write_ark_file(file_name: str, keys: List[str], utterances: List[np.ndarray]):
"""Write utterance matrices to a .ark file"""
"""Write utterance matrices to a .ark file."""
with open(file_name, 'wb') as output_file:
for key, matrix in zip(keys, utterances):
# write a utterance key
@@ -86,7 +86,7 @@ def write_ark_file(file_name: str, keys: List[str], utterances: List[np.ndarray]
def read_utterance_file(file_name: str) -> FileData:
"""Read utterance matrices from a file"""
"""Read utterance matrices from a file."""
file_extension = file_name.split('.')[-1]
if file_extension == 'ark':
@@ -100,7 +100,7 @@ def read_utterance_file(file_name: str) -> FileData:
def write_utterance_file(file_name: str, keys: List[str], utterances: List[np.ndarray]):
"""Write utterance matrices to a file"""
"""Write utterance matrices to a file."""
file_extension = file_name.split('.')[-1]
if file_extension == 'ark':

2
samples/python/speech_sample/speech_sample.py
View File

@@ -22,7 +22,7 @@ from utils import (GNA_ATOM_FREQUENCY, GNA_CORE_FREQUENCY,
def do_inference(data: Dict[str, np.ndarray], infer_request: InferRequest, cw_l: int = 0, cw_r: int = 0) -> np.ndarray:
"""Do a synchronous matrix inference"""
"""Do a synchronous matrix inference."""
frames_to_infer = {}
result = {}

8
samples/python/speech_sample/utils.py
View File

@@ -35,7 +35,7 @@ def compare_with_reference(result: np.ndarray, reference: np.ndarray):
def get_scale_factor(matrix: np.ndarray) -> float:
"""Get scale factor for quantization using utterance matrix"""
"""Get scale factor for quantization using utterance matrix."""
# Max to find scale factor
target_max = 16384
max_val = np.max(matrix)
@@ -46,14 +46,14 @@ def get_scale_factor(matrix: np.ndarray) -> float:
def set_scale_factors(plugin_config: dict, scale_factors: list):
"""Set a scale factor provided for each input"""
"""Set a scale factor provided for each input."""
for i, scale_factor in enumerate(scale_factors):
log.info(f'For input {i} using scale factor of {scale_factor:.7f}')
plugin_config[f'GNA_SCALE_FACTOR_{i}'] = str(scale_factor)
def parse_scale_factors(args: argparse.Namespace) -> list:
"""Get a list of scale factors for input files"""
"""Get a list of scale factors for input files."""
input_files = re.split(', |,', args.input)
scale_factors = re.split(', |,', str(args.scale_factor))
scale_factors = list(map(float, scale_factors))
@@ -72,7 +72,7 @@ def parse_scale_factors(args: argparse.Namespace) -> list:
def parse_outputs_from_args(args: argparse.Namespace) -> Tuple[List[str], List[int]]:
"""Get a list of outputs specified in the args"""
"""Get a list of outputs specified in the args."""
name_and_port = [output.split(':') for output in re.split(', |,', args.output_layers)]
try:
return [name for name, _ in name_and_port], [int(port) for _, port in name_and_port]

48
src/bindings/python/requirements_test.txt
View File

@@ -1,13 +1,35 @@
flake8==3.9.2
flake8-bugbear==21.4.3
flake8-comprehensions==3.3.0
flake8-docstrings==1.6.0
flake8-quotes==3.2.0
mypy==0.812
onnx==1.11.0
pydocstyle==5.1.1
pytest-xdist==2.2.1
pytest==6.1.2
retrying==1.3.3
tox==3.24.5
wheel==0.36.2
bandit
black
flake8
flake8-annotations-complexity
flake8-broken-line
flake8-bugbear
flake8-class-attributes-order
flake8-comprehensions
flake8-debugger
flake8-docstrings
flake8-eradicate
flake8-executable
flake8-expression-complexity
flake8-print
flake8-pytest-style
flake8-rst-docstrings
flake8-string-format
flake8-variables-names
flake8_builtins
flake8_coding
flake8_commas
flake8_pep3101
flake8_quotes
import-order
mypy
onnx
Pep8-naming
pydocstyle
pytest-xdist
pytest
radon
retrying
tox
types-pkg_resources
wheel

62
src/bindings/python/setup.cfg
View File

@@ -1,15 +1,62 @@
[tox:tox]
envlist = py3
[testenv]
skipdist=True
skip_install=True
deps =
-rrequirements.txt
-rrequirements_test.txt
setenv =
OV_BACKEND = {env:OV_BACKEND:"CPU"}
PYTHONPATH = {env:PYTHONPATH}
OpenVINO_DIR = {env:OpenVINO_DIR}
passenv =
http_proxy
https_proxy
commands=
{envbindir}/python setup.py bdist_wheel
{envbindir}/pip install --no-index --pre --find-links=dist/ openvino
pytest --backend={env:OV_BACKEND} tests -v -k 'not _cuda' --ignore=tests/test_onnx/test_zoo_models.py --ignore=tests/test_utils --ignore=tests/test_inference_engine
pytest --backend={env:OV_BACKEND} tests_compatibility/test_ngraph -v -k 'not _cuda' --ignore=tests_compatibility/test_onnx/test_zoo_models.py
[testenv:zoo_models]
commands=
{envbindir}/python setup.py bdist_wheel
{envbindir}/pip install --no-index --pre --find-links=dist/ openvino
pytest --backend={env:OV_BACKEND} tests/test_onnx/test_zoo_models.py -v -n 4 --forked -k 'not _cuda' --model_zoo_xfail
[testenv:devenv]
envdir = devenv
usedevelop = True
deps = -rrequirements.txt
[flake8]
filename = *.py, *.pyx
# ignore:
# D100 - Missing docstring in public module
# D102 - Missing docstring in public method
# D103 - Missing docstring in public function
# D104 - Missing docstring in public package
# D105 - Missing docstring in magic method
# D107 - Missing docstring in __init__
# D412 - No blank lines allowed between a section header and its content
# F401 - module imported but unused
# W503 - line break before binary operator (prefer line breaks before op, not after)
ignore=D100,D102,D103,D104,D105,D107,D412,F401,W503
inline-quotes = double
filename = *.py
max-line-length = 160
ignore = E203
max-parameters-amount = 8
show_source = True
docstring-convention = google
enable-extensions = G
per-file-ignores =
*.pyx: E225, E226, E251, E999, E800, E265, E203, E266, E227, E211
tests/*: S101, T001
*__init__.py: E402, F403, F405, F405
# todo: will be fixed as part of 81803
tests/*: A001,C101,C812,C815,C816,C819,CCE001,D100,D101,D102,D103,D104,D105,D107,D212,E800,ECE001,N400,N802,N806,P101,P103,PT001,PT005,PT006,PT007,PT011,PT012,PT019,PT023,RST201,RST301,S001,T001,VNE001,VNE002,VNE003,W503
tests_compatibility/test_ngraph/*: A001,C101,C812,C815,C816,C819,CCE001,D100,D101,D102,D103,D104,D105,D107,D212,E800,ECE001,N400,N802,N806,P101,P103,PT001,PT005,PT006,PT007,PT011,PT012,PT019,PT023,RST201,RST301,S001,T001,VNE001,VNE002,VNE003,W503
src/compatibility/ngraph/*: A001,A002,C101,C812,C819,CCE001,E800,N803,N806,P101,RST201,RST202,RST203,RST206,RST301,T001,TAE002,VNE001,VNE003
# todo: will be fixed as part of 81803
src/openvino/*: A001,A002,C101,C812,C819,CCE001,E402,E800,N803,N806,P101,RST201,RST202,RST203,RST205,RST206,RST299,RST301,T001,TAE002,VNE001,VNE002,VNE003
[pydocstyle]
convention = google
@@ -21,3 +68,8 @@ show_column_numbers = True
show_error_context = True
show_absolute_path = True
pretty = True
follow_imports=normal
disallow_untyped_defs = True
disallow_untyped_calls = True
check_untyped_defs = True
show_none_errors = True

6
src/bindings/python/src/compatibility/openvino/setup.cfg
View File

@@ -1,16 +1,16 @@
[flake8]
# D104 - Missing docstring in public package
inline-quotes = double
filename = *.py, *.pyx
max-line-length = 160
ignore = E203
ignore = E203,D104
max-parameters-amount = 8
show_source = True
docstring-convention = google
enable-extensions = G
per-file-ignores =
*.pyx: E225, E226, E251, E999, E800, E265, E203, E266, E227, E211
tests/*: S101, T001
*__init__.py: F403, F405, F405
*__init__.py: F403, F405
[pydocstyle]
convention = google

75
src/bindings/python/tox.ini
View File

@@ -1,75 +0,0 @@
[tox]
envlist = py3
[testenv]
skipdist=True
skip_install=True
deps =
-rrequirements.txt
-rrequirements_test.txt
setenv =
NGRAPH_BACKEND = {env:NGRAPH_BACKEND:"CPU"}
PYTHONPATH = {env:PYTHONPATH}
OpenVINO_DIR = {env:OpenVINO_DIR}
passenv =
http_proxy
https_proxy
commands=
{envbindir}/python setup.py bdist_wheel
{envbindir}/pip install --no-index --pre --find-links=dist/ openvino
flake8 {posargs:src/ setup.py} --exclude=src/compatibility/openvino
flake8 --ignore=D100,D101,D102,D103,D104,D105,D107,D212,W503 tests/ tests_compatibility/ --exclude=tests_compatibility/test_inference_engine # ignore lack of docs in tests
mypy --config-file=tox.ini {posargs:src/} --exclude=src/compatibility/openvino
pytest --backend={env:NGRAPH_BACKEND} tests -v -k 'not _cuda' --ignore=tests/test_onnx/test_zoo_models.py --ignore=tests/test_utils --ignore=tests/test_inference_engine
pytest --backend={env:NGRAPH_BACKEND} tests_compatibility/test_ngraph -v -k 'not _cuda' --ignore=tests_compatibility/test_onnx/test_zoo_models.py
[testenv:zoo_models]
commands=
{envbindir}/python setup.py bdist_wheel
{envbindir}/pip install --no-index --pre --find-links=dist/ openvino
pytest --backend={env:NGRAPH_BACKEND} tests/test_onnx/test_zoo_models.py -v -n 4 --forked -k 'not _cuda' --model_zoo_xfail
[testenv:devenv]
envdir = devenv
usedevelop = True
deps = -rrequirements.txt
[flake8]
inline-quotes = "
max-line-length=110
max-complexity=7
# ignore:
# D100 - Missing docstring in public module
# D104 - Missing docstring in public package
# D105 - Missing docstring in magic method
# D107 - Missing docstring in __init__
# D412 - No blank lines allowed between a section header and its content
# F401 - module imported but unused
# W503 - line break before binary operator (prefer line breaks before op, not after)
ignore=D100,D104,D105,D107,D410,D411,D412,F401,W503
[mypy]
ignore_missing_imports=True
follow_imports=normal
disallow_untyped_defs = True
disallow_untyped_calls = True
check_untyped_defs = True
show_error_context = True
show_column_numbers = True
show_none_errors = True
# put custom per-file options here in sections that map their names into filenames, e.g. gta_workflow/filename.py is
# [mypy-ngraph/filename]
[mypy-test.*]
disallow_untyped_defs = False
[isort]
multi_line_output=3
include_trailing_comma=True
force_grid_wrap=0
use_parentheses=True
line_length=100
default_section=FIRSTPARTY
sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
known_third_party=openvino

414
src/bindings/python/wheel/setup.py
View File

@@ -22,118 +22,118 @@ from setuptools.command.build_ext import build_ext
from setuptools.command.build_clib import build_clib
from setuptools.command.install import install
WHEEL_LIBS_INSTALL_DIR = os.path.join('openvino', 'libs')
WHEEL_LIBS_PACKAGE = 'openvino.libs'
PYTHON_VERSION = f'python{sys.version_info.major}.{sys.version_info.minor}'
WHEEL_LIBS_INSTALL_DIR = os.path.join("openvino", "libs")
WHEEL_LIBS_PACKAGE = "openvino.libs"
PYTHON_VERSION = f"python{sys.version_info.major}.{sys.version_info.minor}"
LIBS_DIR = 'bin' if platform.system() == 'Windows' else 'lib'
CONFIG = 'Release' if platform.system() in {'Windows' , 'Darwin'} else ''
LIBS_DIR = "bin" if platform.system() == "Windows" else "lib"
CONFIG = "Release" if platform.system() in {"Windows", "Darwin"} else ""
machine = platform.machine()
if machine == 'x86_64' or machine == 'AMD64':
ARCH = 'intel64'
elif machine == 'X86':
ARCH = 'ia32'
elif machine == 'arm':
ARCH = 'arm'
elif machine == 'aarch64':
ARCH = 'arm64'
if machine == "x86_64" or machine == "AMD64":
ARCH = "intel64"
elif machine == "X86":
ARCH = "ia32"
elif machine == "arm":
ARCH = "arm"
elif machine == "aarch64":
ARCH = "arm64"
# The following variables can be defined in environment or .env file
SCRIPT_DIR = Path(__file__).resolve().parents[0]
CMAKE_BUILD_DIR = os.getenv('CMAKE_BUILD_DIR', '.')
OV_RUNTIME_LIBS_DIR = os.getenv('OV_RUNTIME_LIBS_DIR', f'runtime/{LIBS_DIR}/{ARCH}/{CONFIG}')
TBB_LIBS_DIR = os.getenv('TBB_LIBS_DIR', f'runtime/3rdparty/tbb/{LIBS_DIR}')
PY_PACKAGES_DIR = os.getenv('PY_PACKAGES_DIR', f'python/{PYTHON_VERSION}')
LIBS_RPATH = '$ORIGIN' if sys.platform == 'linux' else '@loader_path'
CMAKE_BUILD_DIR = os.getenv("CMAKE_BUILD_DIR", ".")
OV_RUNTIME_LIBS_DIR = os.getenv("OV_RUNTIME_LIBS_DIR", f"runtime/{LIBS_DIR}/{ARCH}/{CONFIG}")
TBB_LIBS_DIR = os.getenv("TBB_LIBS_DIR", f"runtime/3rdparty/tbb/{LIBS_DIR}")
PY_PACKAGES_DIR = os.getenv("PY_PACKAGES_DIR", f"python/{PYTHON_VERSION}")
LIBS_RPATH = "$ORIGIN" if sys.platform == "linux" else "@loader_path"
LIB_INSTALL_CFG = {
'ie_libs': {
'name': 'core',
'prefix': 'libs.core',
'install_dir': OV_RUNTIME_LIBS_DIR,
'rpath': LIBS_RPATH,
"ie_libs": {
"name": "core",
"prefix": "libs.core",
"install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
},
'hetero_plugin': {
'name': 'hetero',
'prefix': 'libs.core',
'install_dir': OV_RUNTIME_LIBS_DIR,
'rpath': LIBS_RPATH,
"hetero_plugin": {
"name": "hetero",
"prefix": "libs.core",
"install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
},
'gpu_plugin': {
'name': 'gpu',
'prefix': 'libs.core',
'install_dir': OV_RUNTIME_LIBS_DIR,
'rpath': LIBS_RPATH,
"gpu_plugin": {
"name": "gpu",
"prefix": "libs.core",
"install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
},
'cpu_plugin': {
'name': 'cpu',
'prefix': 'libs.core',
'install_dir': OV_RUNTIME_LIBS_DIR,
'rpath': LIBS_RPATH,
"cpu_plugin": {
"name": "cpu",
"prefix": "libs.core",
"install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
},
'multi_plugin': {
'name': 'multi',
'prefix': 'libs.core',
'install_dir': OV_RUNTIME_LIBS_DIR,
'rpath': LIBS_RPATH,
"multi_plugin": {
"name": "multi",
"prefix": "libs.core",
"install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
},
'batch_plugin': {
'name': 'batch',
'prefix': 'libs.core',
'install_dir': OV_RUNTIME_LIBS_DIR,
"batch_plugin": {
"name": "batch",
"prefix": "libs.core",
"install_dir": OV_RUNTIME_LIBS_DIR,
},
'tbb_libs': {
'name': 'tbb',
'prefix': 'libs.tbb',
'install_dir': TBB_LIBS_DIR,
'rpath': LIBS_RPATH,
"tbb_libs": {
"name": "tbb",
"prefix": "libs.tbb",
"install_dir": TBB_LIBS_DIR,
"rpath": LIBS_RPATH,
},
}
PY_INSTALL_CFG = {
'ie_py': {
'name': PYTHON_VERSION,
'prefix': 'site-packages',
'install_dir': PY_PACKAGES_DIR,
"ie_py": {
"name": PYTHON_VERSION,
"prefix": "site-packages",
"install_dir": PY_PACKAGES_DIR,
},
'ngraph_py': {
'name': f'pyngraph_{PYTHON_VERSION}',
'prefix': 'site-packages',
'install_dir': PY_PACKAGES_DIR,
"ngraph_py": {
"name": f"pyngraph_{PYTHON_VERSION}",
"prefix": "site-packages",
"install_dir": PY_PACKAGES_DIR,
},
'pyopenvino' : {
'name': f'pyopenvino_{PYTHON_VERSION}',
'prefix': 'site-packages',
'install_dir': PY_PACKAGES_DIR,
"pyopenvino": {
"name": f"pyopenvino_{PYTHON_VERSION}",
"prefix": "site-packages",
"install_dir": PY_PACKAGES_DIR,
},
}
class PrebuiltExtension(Extension):
"""Initialize Extension"""
"""Initialize Extension."""
def __init__(self, name, sources, *args, **kwargs):
if len(sources) != 1:
nln = '\n'
raise DistutilsSetupError(f'PrebuiltExtension can accept only one source, but got: {nln}{nln.join(sources)}')
nln = "\n"
raise DistutilsSetupError(f"PrebuiltExtension can accept only one source, but got: {nln}{nln.join(sources)}")
super().__init__(name, sources, *args, **kwargs)
class CustomBuild(build):
"""Custom implementation of build_clib"""
"""Custom implementation of build_clib."""
cmake_build_types = ['Release', 'Debug', 'RelWithDebInfo', 'MinSizeRel']
cmake_build_types = ["Release", "Debug", "RelWithDebInfo", "MinSizeRel"]
user_options = [
('config=', None, 'Build configuration [{types}].'.format(types='|'.join(cmake_build_types))),
('jobs=', None, 'Specifies the number of jobs to use with make.'),
('cmake-args=', None, 'Additional options to be passed to CMake.'),
("config=", None, "Build configuration [{types}].".format(types="|".join(cmake_build_types))),
("jobs=", None, "Specifies the number of jobs to use with make."),
("cmake-args=", None, "Additional options to be passed to CMake."),
]
def initialize_options(self):
"""Set default values for all the options that this command supports."""
super().initialize_options()
self.build_base = 'build'
self.build_base = "build"
self.config = None
self.jobs = None
self.cmake_args = None
@@ -144,52 +144,52 @@ class CustomBuild(build):
if not self.config:
if self.debug:
self.config = 'Debug'
self.config = "Debug"
else:
self.announce('Set default value for CMAKE_BUILD_TYPE = Release.', level=4)
self.config = 'Release'
self.announce("Set default value for CMAKE_BUILD_TYPE = Release.", level=4)
self.config = "Release"
else:
build_types = [item.lower() for item in self.cmake_build_types]
try:
i = build_types.index(str(self.config).lower())
self.config = self.cmake_build_types[i]
self.debug = True if 'Debug' == self.config else False
self.debug = True if "Debug" == self.config else False
except ValueError:
self.announce('Unsupported CMAKE_BUILD_TYPE value: ' + self.config, level=4)
self.announce('Supported values: {types}'.format(types=', '.join(self.cmake_build_types)), level=4)
self.announce("Unsupported CMAKE_BUILD_TYPE value: " + self.config, level=4)
self.announce("Supported values: {types}".format(types=", ".join(self.cmake_build_types)), level=4)
sys.exit(1)
if self.jobs is None and os.getenv('MAX_JOBS') is not None:
self.jobs = os.getenv('MAX_JOBS')
if self.jobs is None and os.getenv("MAX_JOBS") is not None:
self.jobs = os.getenv("MAX_JOBS")
self.jobs = multiprocessing.cpu_count() if self.jobs is None else int(self.jobs)
def run(self):
global CMAKE_BUILD_DIR
self.jobs = multiprocessing.cpu_count()
plat_specifier = '.{0}-{1}.{2}'.format(self.plat_name, *sys.version_info[:2])
self.build_temp = os.path.join(self.build_base, 'temp' + plat_specifier, self.config)
plat_specifier = ".{0}-{1}.{2}".format(self.plat_name, *sys.version_info[:2])
self.build_temp = os.path.join(self.build_base, "temp" + plat_specifier, self.config)
# if setup.py is directly called use CMake to build product
if CMAKE_BUILD_DIR == '.':
if CMAKE_BUILD_DIR == ".":
# set path to the root of OpenVINO CMakeList file
openvino_root_dir = Path(__file__).resolve().parents[4]
self.announce(f'Configuring cmake project: {openvino_root_dir}', level=3)
self.spawn(['cmake', '-H' + str(openvino_root_dir), '-B' + self.build_temp,
'-DCMAKE_BUILD_TYPE={type}'.format(type=self.config),
'-DENABLE_PYTHON=ON',
'-DENABLE_OV_ONNX_FRONTEND=ON'])
self.announce(f"Configuring cmake project: {openvino_root_dir}", level=3)
self.spawn(["cmake", "-H" + str(openvino_root_dir), "-B" + self.build_temp,
"-DCMAKE_BUILD_TYPE={type}".format(type=self.config),
"-DENABLE_PYTHON=ON",
"-DENABLE_OV_ONNX_FRONTEND=ON"])
self.announce('Building binaries', level=3)
self.spawn(['cmake', '--build', self.build_temp,
'--config', self.config, '-j', str(self.jobs)])
self.announce("Building binaries", level=3)
self.spawn(["cmake", "--build", self.build_temp,
"--config", self.config, "-j", str(self.jobs)])
CMAKE_BUILD_DIR = self.build_temp
self.run_command('build_clib')
self.run_command("build_clib")
build.run(self)
# Copy extra package_data content filtered by find_packages
dst = Path(self.build_lib)
src = Path(get_package_dir(PY_INSTALL_CFG))
exclude = ignore_patterns('*ez_setup*', '*__pycache__*', '*.egg-info*')
for path in src.glob('**/*'):
exclude = ignore_patterns("*ez_setup*", "*__pycache__*", "*.egg-info*")
for path in src.glob("**/*"):
if path.is_dir() or exclude(str(path)):
continue
path_rel = path.relative_to(src)
@@ -198,7 +198,7 @@ class CustomBuild(build):
class PrepareLibs(build_clib):
"""Prepare prebuilt libraries"""
"""Prepare prebuilt libraries."""
def run(self):
self.configure(LIB_INSTALL_CFG)
@@ -208,30 +208,27 @@ class PrepareLibs(build_clib):
def configure(self, install_cfg):
"""Collect prebuilt libraries. Install them to the temp directories, set rpath."""
for comp, comp_data in install_cfg.items():
install_prefix = comp_data.get('prefix')
install_dir = comp_data.get('install_dir')
install_prefix = comp_data.get("prefix")
install_dir = comp_data.get("install_dir")
if install_dir and not os.path.isabs(install_dir):
install_dir = os.path.join(install_prefix, install_dir)
self.announce(f'Installing {comp}', level=3)
self.spawn(['cmake', '--install', CMAKE_BUILD_DIR, '--prefix', install_prefix, '--component', comp_data.get('name')])
self.announce(f"Installing {comp}", level=3)
self.spawn(["cmake", "--install", CMAKE_BUILD_DIR, "--prefix", install_prefix, "--component", comp_data.get("name")])
# set rpath if applicable
if sys.platform != 'win32' and comp_data.get('rpath'):
file_types = ['.so'] if sys.platform == 'linux' else ['.dylib', '.so']
for path in filter(lambda p: any(item in file_types for item in p.suffixes), Path(install_dir).glob('*')):
set_rpath(comp_data['rpath'], os.path.realpath(path))
if sys.platform != "win32" and comp_data.get("rpath"):
file_types = [".so"] if sys.platform == "linux" else [".dylib", ".so"]
for path in filter(lambda p: any(item in file_types for item in p.suffixes), Path(install_dir).glob("*")):
set_rpath(comp_data["rpath"], os.path.realpath(path))
def generate_package(self, src_dirs):
"""
Collect package data files from preinstalled dirs and
put all runtime libraries to the subpackage
"""
"""Collect package data files from preinstalled dirs and put all runtime libraries to the subpackage."""
# additional blacklist filter, just to fix cmake install issues
blacklist = ['.lib', '.pdb', '_debug.dll', '_debug.dylib']
blacklist = [".lib", ".pdb", "_debug.dll", "_debug.dylib"]
package_dir = os.path.join(get_package_dir(PY_INSTALL_CFG), WHEEL_LIBS_INSTALL_DIR)
for src_dir in src_dirs:
local_base_dir = Path(src_dir)
for file_path in local_base_dir.rglob('*'):
for file_path in local_base_dir.rglob("*"):
file_name = os.path.basename(file_path)
if file_path.is_file() and not any(file_name.endswith(ext) for ext in blacklist):
dst_file = os.path.join(package_dir, os.path.relpath(file_path, local_base_dir))
@@ -239,51 +236,51 @@ class PrepareLibs(build_clib):
copyfile(file_path, dst_file)
if Path(package_dir).exists():
self.announce(f'Adding {WHEEL_LIBS_PACKAGE} package', level=3)
self.announce(f"Adding {WHEEL_LIBS_PACKAGE} package", level=3)
packages.append(WHEEL_LIBS_PACKAGE)
package_data.update({WHEEL_LIBS_PACKAGE: ['*']})
package_data.update({WHEEL_LIBS_PACKAGE: ["*"]})
class CopyExt(build_ext):
"""Copy extension files to the build directory"""
"""Copy extension files to the build directory."""
def run(self):
if len(self.extensions) == 1:
self.run_command('build_clib')
self.run_command("build_clib")
self.extensions = []
self.extensions = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG))
for extension in self.extensions:
if not isinstance(extension, PrebuiltExtension):
raise DistutilsSetupError(f'copy_ext can accept PrebuiltExtension only, but got {extension.name}')
raise DistutilsSetupError(f"copy_ext can accept PrebuiltExtension only, but got {extension.name}")
src = extension.sources[0]
dst = self.get_ext_fullpath(extension.name)
os.makedirs(os.path.dirname(dst), exist_ok=True)
# setting relative path to find dlls
if sys.platform != 'win32':
if sys.platform != "win32":
rpath = os.path.relpath(get_package_dir(PY_INSTALL_CFG), os.path.dirname(src))
if sys.platform == 'linux':
rpath = os.path.join('$ORIGIN', rpath, WHEEL_LIBS_INSTALL_DIR)
elif sys.platform == 'darwin':
rpath = os.path.join('@loader_path', rpath, WHEEL_LIBS_INSTALL_DIR)
if sys.platform == "linux":
rpath = os.path.join("$ORIGIN", rpath, WHEEL_LIBS_INSTALL_DIR)
elif sys.platform == "darwin":
rpath = os.path.join("@loader_path", rpath, WHEEL_LIBS_INSTALL_DIR)
set_rpath(rpath, os.path.realpath(src))
copy_file(src, dst, verbose=self.verbose, dry_run=self.dry_run)
class CustomInstall(install):
"""Enable build_clib during the installation"""
"""Enable build_clib during the installation."""
def run(self):
self.run_command('build')
self.run_command("build")
install.run(self)
class CustomClean(clean):
"""Clean up staging directories"""
"""Clean up staging directories."""
def clean(self, install_cfg):
for comp, comp_data in install_cfg.items():
install_prefix = comp_data.get('prefix')
self.announce(f'Cleaning {comp}: {install_prefix}', level=3)
install_prefix = comp_data.get("prefix")
self.announce(f"Cleaning {comp}: {install_prefix}", level=3)
if os.path.exists(install_prefix):
rmtree(install_prefix)
@@ -294,14 +291,12 @@ class CustomClean(clean):
def ignore_patterns(*patterns):
"""
Filter names by given patterns
"""
"""Filter names by given patterns."""
return lambda name: any(fnmatchcase(name, pat=pat) for pat in patterns)
def is_tool(name):
"""Check if the command-line tool is available"""
"""Check if the command-line tool is available."""
try:
devnull = subprocess.DEVNULL
subprocess.Popen([name], stdout=devnull, stderr=devnull).communicate() # nosec
@@ -312,101 +307,101 @@ def is_tool(name):
def remove_rpath(file_path):
"""
Remove rpath from binaries
"""Remove rpath from binaries.
:param file_path: binary path
:type file_path: pathlib.Path
"""
if sys.platform == 'darwin':
if sys.platform == "darwin":
cmd = (
f'otool -l {file_path} ' # noqa: P103
f'| grep LC_RPATH -A3 '
f'| grep -o "path.*" '
f'| cut -d " " -f2 '
f'| xargs -I{{}} install_name_tool -delete_rpath {{}} {file_path}'
f"otool -l {file_path} " # noqa: P103
f"| grep LC_RPATH -A3 "
f"| grep -o 'path.*' "
f"| cut -d ' ' -f2 "
f"| xargs -I{{}} install_name_tool -delete_rpath {{}} {file_path}"
)
if os.WEXITSTATUS(os.system(cmd)) != 0: # nosec
sys.exit(f'Could not remove rpath for {file_path}')
sys.exit(f"Could not remove rpath for {file_path}")
else:
sys.exit(f'Unsupported platform: {sys.platform}')
sys.exit(f"Unsupported platform: {sys.platform}")
def set_rpath(rpath, executable):
"""Setting rpath for linux and macOS libraries"""
print(f'Setting rpath {rpath} for {executable}') # noqa: T001
"""Setting rpath for linux and macOS libraries."""
print(f"Setting rpath {rpath} for {executable}") # noqa: T001
cmd = []
rpath_tool = ''
rpath_tool = ""
if sys.platform == 'linux':
with open(os.path.realpath(executable), 'rb') as file:
if file.read(1) != b'\x7f':
log.warn(f'WARNING: {executable}: missed ELF header')
if sys.platform == "linux":
with open(os.path.realpath(executable), "rb") as file:
if file.read(1) != b"\x7f":
log.warn(f"WARNING: {executable}: missed ELF header")
return
rpath_tool = 'patchelf'
cmd = [rpath_tool, '--set-rpath', rpath, executable]
elif sys.platform == 'darwin':
rpath_tool = 'install_name_tool'
cmd = [rpath_tool, '-add_rpath', rpath, executable]
rpath_tool = "patchelf"
cmd = [rpath_tool, "--set-rpath", rpath, executable]
elif sys.platform == "darwin":
rpath_tool = "install_name_tool"
cmd = [rpath_tool, "-add_rpath", rpath, executable]
else:
sys.exit(f'Unsupported platform: {sys.platform}')
sys.exit(f"Unsupported platform: {sys.platform}")
if is_tool(rpath_tool):
if sys.platform == 'darwin':
if sys.platform == "darwin":
remove_rpath(executable)
ret_info = subprocess.run(cmd, check=True, shell=False) # nosec
if ret_info.returncode != 0:
sys.exit(f'Could not set rpath: {rpath} for {executable}')
sys.exit(f"Could not set rpath: {rpath} for {executable}")
else:
sys.exit(f'Could not found {rpath_tool} on the system, ' f'please make sure that this tool is installed')
sys.exit(f"Could not found {rpath_tool} on the system, " f"please make sure that this tool is installed")
def find_prebuilt_extensions(search_dirs):
"""collect prebuilt python extensions"""
"""Collect prebuilt python extensions."""
extensions = []
ext_pattern = ''
if sys.platform == 'linux':
ext_pattern = '**/*.so'
elif sys.platform == 'win32':
ext_pattern = '**/*.pyd'
elif sys.platform == 'darwin':
ext_pattern = '**/*.so'
ext_pattern = ""
if sys.platform == "linux":
ext_pattern = "**/*.so"
elif sys.platform == "win32":
ext_pattern = "**/*.pyd"
elif sys.platform == "darwin":
ext_pattern = "**/*.so"
for base_dir in search_dirs:
for path in Path(base_dir).glob(ext_pattern):
if path.match('openvino/libs/*'):
if path.match("openvino/libs/*"):
continue
relpath = path.relative_to(base_dir)
if relpath.parent != '.':
if relpath.parent != ".":
package_names = str(relpath.parent).split(os.path.sep)
else:
package_names = []
package_names.append(path.name.split('.', 1)[0])
name = '.'.join(package_names)
package_names.append(path.name.split(".", 1)[0])
name = ".".join(package_names)
extensions.append(PrebuiltExtension(name, sources=[str(path)]))
if not extensions:
extensions.append(PrebuiltExtension('openvino', sources=[str('setup.py')]))
extensions.append(PrebuiltExtension("openvino", sources=[str("setup.py")]))
return extensions
def get_description(desc_file_path):
"""read description from README.md"""
with open(desc_file_path, 'r', encoding='utf-8') as fstream:
"""Read description from README.md."""
with open(desc_file_path, "r", encoding="utf-8") as fstream:
description = fstream.read()
return description
def get_dependencies(requirements_file_path):
"""read dependencies from requirements.txt"""
with open(requirements_file_path, 'r', encoding='utf-8') as fstream:
"""Read dependencies from requirements.txt."""
with open(requirements_file_path, "r", encoding="utf-8") as fstream:
dependencies = fstream.read()
return dependencies
def get_dir_list(install_cfg):
"""collect all available directories with libs or python packages"""
"""Collect all available directories with libs or python packages."""
dirs = []
for comp_info in install_cfg.values():
cfg_prefix = comp_info.get('prefix')
cfg_dir = comp_info.get('install_dir')
cfg_prefix = comp_info.get("prefix")
cfg_dir = comp_info.get("install_dir")
if cfg_dir:
if not os.path.isabs(cfg_dir):
cfg_dir = os.path.join(cfg_prefix, cfg_dir)
@@ -416,11 +411,8 @@ def get_dir_list(install_cfg):
def get_package_dir(install_cfg):
"""
Get python package path based on config
All the packages should be located in one directory
"""
py_package_path = ''
"""Get python package path based on config. All the packages should be located in one directory."""
py_package_path = ""
dirs = get_dir_list(install_cfg)
if len(dirs) != 0:
# setup.py support only one package directory, all modules should be located there
@@ -429,62 +421,62 @@ def get_package_dir(install_cfg):
def concat_files(output_file, input_files):
with open(output_file, 'w', encoding='utf-8') as outfile:
with open(output_file, "w", encoding="utf-8") as outfile:
for filename in input_files:
with open(filename, 'r', encoding='utf-8') as infile:
with open(filename, "r", encoding="utf-8") as infile:
content = infile.read()
outfile.write(content)
return output_file
platforms = ['linux', 'win32', 'darwin']
platforms = ["linux", "win32", "darwin"]
if not any(pl in sys.platform for pl in platforms):
sys.exit(f'Unsupported platform: {sys.platform}, expected: linux, win32, darwin')
sys.exit(f"Unsupported platform: {sys.platform}, expected: linux, win32, darwin")
# copy license file into the build directory
package_license = os.getenv('WHEEL_LICENSE', SCRIPT_DIR.parents[3] / 'LICENSE')
package_license = os.getenv("WHEEL_LICENSE", SCRIPT_DIR.parents[3] / "LICENSE")
if os.path.exists(package_license):
copyfile(package_license, 'LICENSE')
copyfile(package_license, "LICENSE")
packages = find_namespace_packages(get_package_dir(PY_INSTALL_CFG))
package_data: typing.Dict[str, list] = {}
pkg_name = os.getenv('WHEEL_PACKAGE_NAME', 'openvino')
ext_modules = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) if pkg_name == 'openvino' else []
pkg_name = os.getenv("WHEEL_PACKAGE_NAME", "openvino")
ext_modules = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) if pkg_name == "openvino" else []
description_md = SCRIPT_DIR.parents[3] / 'docs' / 'install_guides' / 'pypi-openvino-rt.md'
md_files = [description_md, SCRIPT_DIR.parents[3] / 'docs' / 'install_guides' / 'pre-release-note.md']
docs_url = 'https://docs.openvino.ai/latest/index.html'
description_md = SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pypi-openvino-rt.md"
md_files = [description_md, SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pre-release-note.md"]
docs_url = "https://docs.openvino.ai/latest/index.html"
if(os.getenv('CI_BUILD_DEV_TAG')):
output = Path.cwd() / 'build' / 'pypi-openvino-rt.md'
if(os.getenv("CI_BUILD_DEV_TAG")):
output = Path.cwd() / "build" / "pypi-openvino-rt.md"
output.parent.mkdir(exist_ok=True)
description_md = concat_files(output, md_files)
docs_url = 'https://docs.openvino.ai/nightly/index.html'
docs_url = "https://docs.openvino.ai/nightly/index.html"
setup(
version=os.getenv('WHEEL_VERSION', '0.0.0'),
build=os.getenv('WHEEL_BUILD', '000'),
author_email=os.getenv('WHEEL_AUTHOR_EMAIL', 'openvino_pushbot@intel.com'),
version=os.getenv("WHEEL_VERSION", "0.0.0"),
build=os.getenv("WHEEL_BUILD", "000"),
author_email=os.getenv("WHEEL_AUTHOR_EMAIL", "openvino_pushbot@intel.com"),
name=pkg_name,
license=os.getenv('WHEEL_LICENCE_TYPE', 'OSI Approved :: Apache Software License'),
author=os.getenv('WHEEL_AUTHOR', 'Intel(R) Corporation'),
description=os.getenv('WHEEL_DESC', 'OpenVINO(TM) Runtime'),
install_requires=get_dependencies(os.getenv('WHEEL_REQUIREMENTS', SCRIPT_DIR.parents[0] / 'requirements.txt')),
long_description=get_description(os.getenv('WHEEL_OVERVIEW', description_md)),
long_description_content_type='text/markdown',
download_url=os.getenv('WHEEL_DOWNLOAD_URL', 'https://github.com/openvinotoolkit/openvino/tags'),
url=os.getenv('WHEEL_URL', docs_url),
license=os.getenv("WHEEL_LICENCE_TYPE", "OSI Approved :: Apache Software License"),
author=os.getenv("WHEEL_AUTHOR", "Intel(R) Corporation"),
description=os.getenv("WHEEL_DESC", "OpenVINO(TM) Runtime"),
install_requires=get_dependencies(os.getenv("WHEEL_REQUIREMENTS", SCRIPT_DIR.parents[0] / "requirements.txt")),
long_description=get_description(os.getenv("WHEEL_OVERVIEW", description_md)),
long_description_content_type="text/markdown",
download_url=os.getenv("WHEEL_DOWNLOAD_URL", "https://github.com/openvinotoolkit/openvino/tags"),
url=os.getenv("WHEEL_URL", docs_url),
cmdclass={
'build': CustomBuild,
'install': CustomInstall,
'build_clib': PrepareLibs,
'build_ext': CopyExt,
'clean': CustomClean,
"build": CustomBuild,
"install": CustomInstall,
"build_clib": PrepareLibs,
"build_ext": CopyExt,
"clean": CustomClean,
},
ext_modules=ext_modules,
packages=packages,
package_dir={'': get_package_dir(PY_INSTALL_CFG)},
package_dir={"": get_package_dir(PY_INSTALL_CFG)},
package_data=package_data,
zip_safe=False,
)