IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add watchdog of OpenVino ONNX CI (#2550)

Browse Source
This commit is contained in:
Rafal Blaczkowski
2020-10-23 14:16:43 +02:00
committed by GitHub
parent 9c78a4855a
commit 46e8c12a5d
7 changed files with 1000 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
.ci/openvino-onnx/watchdog/Jenkinsfile vendored Normal file
View File

@@ -0,0 +1,65 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
timeout(30)
{
node(LABEL) {
BUILD_WORKSPACE = "$WORKSPACE/$BUILD_NUMBER"
WATCHDOG_ROOT = "$BUILD_WORKSPACE/.ci/openvino-onnx/watchdog"
VENV_PATH = "${BUILD_WORKSPACE}/.wdvenv"
try {
stage("Clone repository") {
dir ("$BUILD_WORKSPACE") {
checkout([$class: 'GitSCM', branches: [[name: "*/$BRANCH"]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${GITHUB_KEY}", url: 'git@github.com:rblaczkowski/openvino.git']]])
}
}
stage("Prepare environment") {
sh """#!/bin/bash
if [ ! -d ${VENV_PATH} ]; then
python3 -m venv ${VENV_PATH}
source ${VENV_PATH}/bin/activate
pip install -r ${WATCHDOG_ROOT}/requirements.txt
fi
"""
}
stage("Run script") {
withCredentials([
usernamePassword(credentialsId: '7157091e-bc04-42f0-99fd-dc4da2922a55',
usernameVariable: 'username',
passwordVariable: 'password')])
{
dir ("$BUILD_WORKSPACE") {
sh """#!/bin/bash
source ${VENV_PATH}/bin/activate
export PYTHONHTTPSVERIFY=0
python ${WATCHDOG_ROOT}/src/main.py \
--msteams-url=${MSTEAMS_URL_FILE} \
--github-credentials '${username}' '${password}' \
--github-org=${GITHUB_ORG} \
--github-project=${GITHUB_PROJECT} \
--jenkins-token=${JENKINS_TOKEN_FILE} \
--jenkins-server=${JENKINS_SERVER} \
--jenkins-user=${JENKINS_USER} \
--ci-job=${CI_JOB_NAME} \
--watchdog-job=${WATCHDOG_JOB_NAME}
"""
}
}
}
} catch (e) {
echo "$e"
currentBuild.result = "FAILURE"
} finally {
stage("Cleanup") {
sh """
cd $BUILD_WORKSPACE
rm -rf ..?* .[!.]* *
"""
}
}
}
}

6
.ci/openvino-onnx/watchdog/requirements.txt Normal file
View File

@@ -0,0 +1,6 @@
python-jenkins==1.7.0
retrying==1.3.3
pygithub==1.51
timeout-decorator==0.4.1
requests==2.23.0
wheel

117
.ci/openvino-onnx/watchdog/src/git_wrapper.py Normal file
View File

@@ -0,0 +1,117 @@
#!/usr/bin/python3
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import logging
import timeout_decorator
from datetime import datetime
from retrying import retry
from github import Github, GithubException
# Logging
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
_RETRY_LIMIT = 3
_RETRY_COOLDOWN_MS = 2000
_REQUEST_TIMEOUT_S = 10
class GitWrapper:
"""Class wrapping PyGithub API.
The purpose of this class is to wrap methods from PyGithub API used in Watchdog, for less error-prone and
more convenient use. Docs for used API, including wrapped methods can be found at:
https://pygithub.readthedocs.io/en/latest/introduction.html
:param github_credentials: Credentials used for GitHub
:param repository: GitHub repository name
:param project: GitHub project name
:type github_credentials: String
:type repository: String
:type project: String
"""
def __init__(self, github_credentials, repository, project):
self.git = Github(*github_credentials)
self.repository = repository
self.project = project
self.github_credentials = github_credentials
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_git_time(self):
"""Retrieve time from GitHub.
Used to reliably determine time during Watchdog run.
:return: Datetime object describing current time
:rtype: datetime
"""
try:
datetime_object = self._get_git_time()
except ValueError as e:
raise GitWrapperError(str(e))
except GithubException as e:
message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e))
raise GitWrapperError(message)
except timeout_decorator.TimeoutError:
message = 'GitHub Exception during API status retrieval. Timeout during API request.'
raise GitWrapperError(message)
return datetime_object
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_pull_requests(self):
"""Retrieve paginated list of pull requests from GitHub.
:return: Paginated list of Pull Requests in GitHub repo
:rtype: github.PaginatedList.PaginatedList of github.PullRequest.PullRequest
"""
try:
prs = self._get_pull_requests()
except GithubException as e:
message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e))
raise GitWrapperError(message)
return prs
@timeout_decorator.timeout(_REQUEST_TIMEOUT_S)
def _get_git_time(self):
"""Private method retrieving time from GitHub.
:return: Datetime object describing current time
:rtype: datetime
"""
datetime_string = self.git.get_api_status().raw_headers.get('date', '')
datetime_format = '%a, %d %b %Y %H:%M:%S %Z'
datetime_object = datetime.strptime(datetime_string, datetime_format)
return datetime_object
@timeout_decorator.timeout(_REQUEST_TIMEOUT_S)
def _get_pull_requests(self):
"""Private method retrieving pull requests from GitHub.
:return: Paginated list of Pull Requests in GitHub repo
:rtype: github.PaginatedList.PaginatedList of github.PullRequest.PullRequest
"""
message = "git: {}".format(self.git)
message2 = "repository: {}".format(self.repository)
message3 = "project: {}".format(self.project)
message4 = "github_credentials: {}".format(self.github_credentials)
log.warning(message)
log.warning(message2)
log.warning(message3)
log.warning(message4)
return self.git.get_organization(self.repository).get_repo(self.project).get_pulls()
class GitWrapperError(Exception):
"""Base class for exceptions raised in GitWrapper.
:param message Explanation of the error
"""
def __init__(self, message):
self.message = message
log.exception(message)

91
.ci/openvino-onnx/watchdog/src/jenkins_wrapper.py Normal file
View File

@@ -0,0 +1,91 @@
#!/usr/bin/python3
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import requests
import jenkins
import logging
from retrying import retry
# Logging
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
_RETRY_LIMIT = 3
_RETRY_COOLDOWN_MS = 5000
class JenkinsWrapper:
"""Class wrapping Python-Jenkins API.
The purpose of this class is to wrap methods from Python-Jenkins API used in Watchdog, for less error-prone and
more convenient use. Docs for used API, including wrapped methods can be found at:
https://python-jenkins.readthedocs.io/en/latest/
:param jenkins_token: Token used for Jenkins
:param jenkins_user: Username used to connect to Jenkins
:param jenkins_server: Jenkins server address
:type jenkins_token: String
:type jenkins_user: String
:type jenkins_server: String
"""
def __init__(self, jenkins_token, jenkins_user, jenkins_server):
self.jenkins_server = jenkins_server
self.jenkins = jenkins.Jenkins(jenkins_server, username=jenkins_user,
password=jenkins_token)
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_build_console_output(self, job_name, build_number):
return self.jenkins.get_build_console_output(job_name, build_number)
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_job_info(self, job_name):
return self.jenkins.get_job_info(job_name)
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_build_info(self, job_name, build_number):
return self.jenkins.get_build_info(job_name, build_number)
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_queue_item(self, queue_id):
"""Attempt to retrieve Jenkins job queue item.
Exception communicating queue doesn't exist is expected,
in that case method returns empty dict.
:param queue_id: Jenkins job queue ID number
:type queue_id: int
:return: Dictionary representing Jenkins job queue item
:rtype: dict
"""
try:
return self.jenkins.get_queue_item(queue_id)
except Exception as e:
# Exception 'queue does not exist' is expected behaviour when job is running
if 'queue' in str(e) and 'does not exist' in str(e):
return {}
else:
raise
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
def get_idle_ci_hosts(self):
"""Query Jenkins for idle servers.
Send GET request to Jenkins server, querying for idle servers labeled
for OpenVino-ONNX CI job.
:return: Number of idle hosts delegated to OpenVino-ONNX CI
:rtype: int
"""
jenkins_request_url = self.jenkins_server + 'label/ci&&onnx/api/json?pretty=true'
try:
log.info('Sending request to Jenkins: %s', jenkins_request_url)
r = requests.Request(method='GET', url=jenkins_request_url, verify=False)
response = self.jenkins.jenkins_request(r).json()
return int(response['totalExecutors']) - int(response['busyExecutors'])
except Exception as e:
log.exception('Failed to send request to Jenkins!\nException message: %s', str(e))
raise

89
.ci/openvino-onnx/watchdog/src/main.py Normal file
View File

@@ -0,0 +1,89 @@
#!/usr/bin/python3
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import argparse
import sys
from watchdog import Watchdog
DEFAULT_MSTEAMS_URL_FILE = '/home/lab_nerval/tokens/msteams_url'
DEFAULT_GITHUB_ORGANIZATION = 'openvinotoolkit'
DEFAULT_GITHUB_PROJECT = 'openvino'
DEFAULT_JENKINS_TOKEN_FILE = '/home/lab_nerval/tokens/crackerjack'
DEFAULT_JENKINS_SERVER = 'https://crackerjack.intel.com/'
DEFAULT_JENKINS_USER = 'lab_nerval'
DEFAULT_CI_JOB_NAME = 'onnx/OpenVino_CI'
DEFAULT_WATCHDOG_JOB_NAME = 'onnx/ci_watchdog'
def main(args):
"""
Read args passed to script, load tokens and run watchdog.
Keyword arguments:
:param args: arguments parsed by argparse ArgumentParser
:return: returns status code 0 on successful completion
"""
jenkins_server = args.jenkins_server.strip()
jenkins_user = args.jenkins_user.strip()
jenkins_token = open(args.jenkins_token).read().replace('\n', '').strip()
msteams_url = open(args.msteams_url).read().replace('\n', '').strip()
github_credentials = args.github_credentials
github_org = args.github_org
github_project = args.github_project
ci_job = args.ci_job.strip()
watchdog_job = args.watchdog_job.strip()
quiet = args.quiet
wd = Watchdog(jenkins_token=jenkins_token,
jenkins_server=jenkins_server,
jenkins_user=jenkins_user,
github_credentials=github_credentials,
git_org=github_org,
git_project=github_project,
msteams_url=msteams_url,
ci_job_name=ci_job,
watchdog_job_name=watchdog_job)
wd.run(quiet=quiet)
return 0
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--msteams-url', help='Path to MS Teams channel url to communicate messages.',
default=DEFAULT_MSTEAMS_URL_FILE, action='store', required=False)
parser.add_argument('--github-credentials', help='GitHub user credentials to access repo.',
nargs="+", required=True)
parser.add_argument('--github-org', help='Name of organization on GitHub.',
default=DEFAULT_GITHUB_ORGANIZATION, action='store', required=False)
parser.add_argument('--github-project', help='Name of project on GitHub.',
default=DEFAULT_GITHUB_PROJECT, action='store', required=False)
parser.add_argument('--jenkins-token', help='Path to Jenkins user token to access build info.',
default=DEFAULT_JENKINS_TOKEN_FILE, action='store', required=False)
parser.add_argument('--jenkins-server', help='Jenkins server address.',
default=DEFAULT_JENKINS_SERVER, action='store', required=False)
parser.add_argument('--jenkins-user', help='Jenkins user used to log in.',
default=DEFAULT_JENKINS_USER, action='store', required=False)
parser.add_argument('--ci-job', help='Jenkins CI job name.',
default=DEFAULT_CI_JOB_NAME, action='store', required=False)
parser.add_argument('--watchdog-job', help='Jenkins CI Watchdog job name.',
default=DEFAULT_WATCHDOG_JOB_NAME, action='store', required=False)
parser.add_argument('--quiet', help="Quiet mode - doesn\'t send message to communicator.",
action='store_true', required=False)
args = parser.parse_args()
sys.exit(main(args))

128
.ci/openvino-onnx/watchdog/src/ms_teams_communicator.py Normal file
View File

@@ -0,0 +1,128 @@
#!/usr/bin/python3
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import requests
class MSTeamsCommunicator:
"""Class communicating with MSTeams using Incoming Webhook.
The purpose of this class is to use MSTeams API to send message.
Docs for used API, including wrapped methods can be found at:
https://docs.microsoft.com/en-us/outlook/actionable-messages/send-via-connectors
"""
def __init__(self, _ci_alerts_channel_url):
self._ci_alerts_channel_url = _ci_alerts_channel_url
self._queued_messages = {
self._ci_alerts_channel_url: [],
}
@property
def messages(self):
"""
Get list of queued messages.
:return: List of queued messages
:return type: List[String]
"""
return self._queued_messages.values()
def queue_message(self, message):
"""
Queue message to be sent later.
:param message: Message content
:type message: String
"""
self._queued_messages[self._ci_alerts_channel_url].append(message)
def _parse_text(self, watchdog_log, message):
"""
Parse text to display as alert.
:param watchdog_log: Watchdog log content
:param message: Unparsed message content
:type watchdog_log: String
:type message: String
"""
message_split = message.split('\n')
log_url = None
if len(message_split) == 3:
log_url = message_split[-1]
title = message_split[0]
text = message_split[1]
header = watchdog_log.split(' - ')
header_formatted = '{} - [Watchdog Log]({})'.format(header[0], header[1])
return title, log_url, '{}\n\n{}'.format(header_formatted, text)
def _json_request_content(self, title, log_url, text_formatted):
"""
Create final json request to send message to MS Teams channel.
:param title: Title of alert
:param log_url: URL to PR
:param text_formatted: General content of alert - finally formatted
:type title: String
:type title: String
:type title: String
"""
data = {
'@context': 'https://schema.org/extensions',
'@type': 'MessageCard',
'themeColor': '0072C6',
'title': title,
'text': text_formatted,
'potentialAction':
[
{
'@type': 'OpenUri',
'name': 'Open PR',
'targets':
[
{
'os': 'default',
'uri': log_url,
},
],
},
],
}
return data
def _send_to_channel(self, watchdog_log, message_queue, channel_url):
"""
Send MSTeams message to specified channel.
:param watchdog_log: Watchdog log content
:param message_queue: Queued messages to send
:param channel_url: Channel url
:type watchdog_log: String
:type message_queue: String
:type channel_url: String
"""
for message in message_queue:
title, log_url, text_formatted = self._parse_text(watchdog_log, message)
data = self._json_request_content(title, log_url, text_formatted)
try:
requests.post(url=channel_url, json=data)
except Exception as ex:
raise Exception('!!CRITICAL!! MSTeamsCommunicator: Could not send message '
'due to {}'.format(ex))
def send_message(self, watchdog_log, quiet=False):
"""
Send queued messages as single communication.
:param watchdog_log: Watchdog log content
:param quiet: Flag for disabling sending report through MS Teams
:type watchdog_log: String
:type quiet: Boolean
"""
for channel, message_queue in self._queued_messages.items():
if not quiet and message_queue:
self._send_to_channel(watchdog_log, message_queue, channel)

504
.ci/openvino-onnx/watchdog/src/watchdog.py Normal file
View File

@@ -0,0 +1,504 @@
#!/usr/bin/python3
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import datetime
import time
import re
import logging
import requests
from ms_teams_communicator import MSTeamsCommunicator
from jenkins_wrapper import JenkinsWrapper
from jenkins import NotFoundException
from git_wrapper import GitWrapper, GitWrapperError
import os
import json
# Logging
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
# Watchdog static constant variables
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
_BUILD_DURATION_THRESHOLD = datetime.timedelta(minutes=60)
_CI_START_THRESHOLD = datetime.timedelta(minutes=30)
_AWAITING_JENKINS_THRESHOLD = datetime.timedelta(minutes=5)
_WATCHDOG_DIR = os.path.expanduser('~')
_PR_REPORTS_CONFIG_KEY = 'pr_reports'
_CI_BUILD_FAIL_MESSAGE = 'ERROR: py3: commands failed'
_CI_BUILD_SUCCESS_MESSAGE = 'py3: commands succeeded'
_GITHUB_CI_CHECK_NAME = 'OpenVINO-ONNX'
INTERNAL_ERROR_MESSAGE_HEADER = '!!! --- !!! INTERNAL WATCHDOG ERROR !!! --- !!!'
ERROR_MESSAGE_HEADER = '!!! OpenVino-ONNX CI Error !!!'
WARNING_MESSAGE_HEADER = 'OpenVino-ONNX CI WARNING'
INFO_MESSAGE_HEADER = 'OpenVino-ONNX CI INFO'
class Watchdog:
"""Class describing OpenVino-ONNX-CI Watchdog.
Watchdog connects to GitHub and retrieves the list of current pull requests (PRs) in
OpenVino repository. Then it connects to specified Jenkins server to
check CI jobs associated with every PR. Watchdog verifies time durations for Jenkins
initial response, job queue and execution against time treshold constants. Every fail
is logged and reported through MS Teams communicators.
:param jenkins_token: Token used for Jenkins
:param jenkins_server: Jenkins server address
:param jenkins_user: Username used to connect to Jenkins
:param github_credentials: Credentials used to connect to GitHub
:param msteams_url: URL used to connect to MS Teams channel
:param ci_job_name: OpenVino-ONNX CI job name used in Jenkins
:param watchdog_job_name: Watchdog job name used in Jenkins
:type jenkins_token: String
:type jenkins_server: String
:type jenkins_user: String
:type github_credentials: String
:type msteams_url: String
:type ci_job_name: String
:type watchdog_job_name: String
.. note::
Watchdog and OpenVino-ONNX CI job must be placed on the same Jenkins server.
"""
def __init__(self, jenkins_token, jenkins_server, jenkins_user, github_credentials, git_org,
git_project, msteams_url, ci_job_name, watchdog_job_name):
self._config_path = os.path.join(_WATCHDOG_DIR, '{}/.{}_ci_watchdog.json'.format(_WATCHDOG_DIR, git_project))
# Jenkins Wrapper object for CI job
self._jenkins = JenkinsWrapper(jenkins_token,
jenkins_user=jenkins_user,
jenkins_server=jenkins_server)
# Load GitHub token and log in, retrieve pull requests
self._git = GitWrapper(github_credentials, repository=git_org, project=git_project)
# Create MS Teams api object
self._msteams_hook = MSTeamsCommunicator(msteams_url)
self._ci_job_name = ci_job_name.lower()
self._watchdog_job_name = watchdog_job_name
# Read config file
self._config = self._read_config_file()
# Time at Watchdog initiation
self._now_time = datetime.datetime.now()
self._current_prs = {}
self._ms_teams_enabled = True
def run(self, quiet=False):
"""Run main watchdog logic.
Retrieve list of pull requests and pass it to the method responsible for checking them.
:param quiet: Flag for disabling sending report through communicator
:type quiet: Boolean
"""
try:
pull_requests = self._git.get_pull_requests()
except GitWrapperError:
message = 'Failed to retrieve Pull Requests!'
log.exception(message)
self._queue_message(message, message_severity='internal')
# Check all pull requests
for pr in pull_requests:
try:
self._check_pr(pr)
except Exception as e:
log.exception(str(e))
self._queue_message(str(e), message_severity='internal', pr=pr)
self._update_config()
self._send_message(quiet=quiet)
def _read_config_file(self):
"""Read Watchdog config file stored on the system.
The file stores every fail already reported along with timestamp. This
mechanism is used to prevent Watchdog from reporting same failure
multiple times. In case there's no config under the expected path,
appropriate data structure is created and returned.
:return: Returns dict of dicts with reported fails with their timestamps
:rtype: dict of dicts
"""
if os.path.isfile(self._config_path):
log.info('Reading config file in: {}'.format(self._config_path))
file = open(self._config_path, 'r')
data = json.load(file)
else:
log.info('No config file found in: {}'.format(self._config_path))
data = {_PR_REPORTS_CONFIG_KEY: {}}
return data
def _check_pr(self, pr):
"""Check pull request (if there's no reason to skip).
Retrieve list of statuses for every PR's last commit and interpret them. Filters out statuses
unrelated to OpenVino-ONNX Jenkins CI and passes relevant statuses to method that interprets them.
If no commit statuses related to Jenkins are available after time defined by
**_AWAITING_JENKINS_THRESHOLD** calls appropriate method to check for builds waiting in queue.
:param pr: GitHub Pull Requests
:type pr: github.PullRequest.PullRequest
"""
log.info('===============================================')
log.info('Checking PR#{}'.format(pr.number))
# Get last Jenkins status
last_status = self._get_last_status(pr)
# Append PR checked in current run for Watchdog config
self._current_prs[str(pr.number)] = self._get_pr_timestamps(pr, last_status)
if self._should_ignore(pr) or self._updated_since_last_run(pr):
log.info('Ignoring PR#{}'.format(pr.number))
return
# Calculate time passed since PR update (any commit, merge or comment)
pr_time_delta = self._now_time - pr.updated_at
if last_status:
# Interpret found CI statuses
log.info('Last status: {} at {}'.format(last_status.description, last_status.updated_at))
self._interpret_status(last_status, pr)
elif pr_time_delta > _CI_START_THRESHOLD:
# If there's no status after assumed time - check if build is waiting in queue
log.info('CI for PR {}: NO JENKINS STATUS YET'.format(pr.number))
self._check_missing_status(pr)
@staticmethod
def _get_pr_timestamps(pr, last_status):
"""Get dict containing PR timestamp and last status timestamp.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
:return: Dictionary with PR and last status update timestamps
:rtype: dict
"""
pr_timestamp = time.mktime(pr.updated_at.timetuple())
if last_status:
status_timestamp = time.mktime(last_status.updated_at.timetuple())
else:
status_timestamp = None
pr_dict = {'pr_timestamp': pr_timestamp,
'status_timestamp': status_timestamp}
return pr_dict
@staticmethod
def _get_last_status(pr):
"""Get last commit status posted from Jenkins.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
:return: Either last PR status posted from Jenkins or None
:rtype: github.CommitStatus.CommitStatus
"""
# Find last commit in PR
last_commit = pr.get_commits().reversed[0]
# Get statuses and filter them to contain only those related to Jenkins CI
# and check if CI in Jenkins started
statuses = last_commit.get_statuses()
jenk_statuses = [stat for stat in statuses if
_GITHUB_CI_CHECK_NAME in stat.context]
try:
last_status = jenk_statuses[0]
except IndexError:
last_status = None
return last_status
@staticmethod
def _should_ignore(pr):
"""Determine if PR should be ignored.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
:return: Returns True if PR should be ignored
:rtype: Bool
"""
# Ignore PR if it has WIP label or WIP in title
if 'WIP' in pr.title:
log.info('PR#{} should be ignored. WIP tag in title.'.format(pr.number))
return True
label_names = [label.name for label in pr.labels]
if 'WIP' in label_names:
log.info('PR#{} should be ignored. WIP label present.'.format(pr.number))
return True
# Ignore PR if base ref is not master
if 'master' not in pr.base.ref:
log.info('PR#{} should be ignored. Base ref is not master'.format(pr.number))
return True
# Ignore PR if mergeable state is 'dirty' or 'behind'.
# Practically this ignores PR in case of merge conflicts
ignored_mergeable_states = ['behind', 'dirty', 'draft']
if pr.mergeable_state in ignored_mergeable_states:
log.info('PR#{} should be ignored. Mergeable state is {}. '.format(pr.number, pr.mergeable_state))
return True
# If no criteria for ignoring PR are met - return false
return False
def _updated_since_last_run(self, pr):
# Ignore if PR was already checked and there was no update in meantime
pr_number = str(pr.number)
current_pr_timestamps = self._current_prs.get(pr_number)
last_pr_timestamps = self._config[_PR_REPORTS_CONFIG_KEY].get(pr_number)
if current_pr_timestamps == last_pr_timestamps:
log.info('PR#{} - No update since last check'.format(pr.number))
return True
else:
return False
def _check_missing_status(self, pr):
"""Verify if missing status is expected.
This method checks if CI build for last was scheduled and still waits in queue for
executor.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
"""
pr_time_delta = self._now_time - pr.updated_at
try:
build_number = self._build_scheduled(pr)
if self._build_in_queue(pr, build_number):
message = ('PR# {}: build waiting in queue after {} minutes.'
.format(pr.number, pr_time_delta.seconds / 60))
severity = 'warning'
else:
message = ('PR# {}: missing status on GitHub after {} minutes.'
.format(pr.number, pr_time_delta.seconds / 60))
severity = 'error'
self._queue_message(message, message_severity=severity, pr=pr)
except TypeError:
log.info('Committer outside of OpenVino organization')
def _build_scheduled(self, pr):
"""Check if Jenkins build corresponding to PR was scheduled.
This method takes last Jenkins build for given PR and compares hash from Jenkins console output
and sha from PR object to determine if CI build for appropriate commit was scheduled.
:param pr: Single PR being currently checked
:type pr: github.PullRequest.PullRequest
:return: Returns build number or -1 if no build found
:rtype: int
"""
pr_number = str(pr.number)
project_name_full = self._ci_job_name + '/PR-' + pr_number
try:
# Retrieve console output from last Jenkins build for job corresponding to this PR
last_build_number = self._jenkins.get_job_info(project_name_full)['lastBuild']['number']
console_output = self._jenkins.get_build_console_output(project_name_full, last_build_number)
# Check if CI build was scheduled - commit hash on GH must match hash in last Jenkins build console output
# Retrieve hash from Jenkins output
match_string = '(?:Obtained .ci/[a-zA-Z/]+Jenkinsfile from ([a-z0-9]{40}))'
retrieved_sha = re.search(match_string, console_output).group(1)
if retrieved_sha == pr.get_commits().reversed[0].sha:
return last_build_number
else:
return -1
except (NotFoundException, AttributeError, requests.exceptions.HTTPError):
message = ('PR #{}: Jenkins build corresponding to commit {} not found!'
.format(pr_number, pr.get_commits().reversed[0].sha))
self._queue_message(message, message_severity='error', pr=pr)
return -1
def _build_in_queue(self, pr, build_number):
"""Check if Jenkins build waits in queue.
This method verifies if CI build is waiting in queue based on console output.
:param pr: Single PR being currently checked
:param build_number: Jenkins build number to retrieve console output from
:type pr: github.PullRequest.PullRequest
:type build_number: int
:return: Returns True if CI build is waiting in queue
:rtype: Bool
"""
pr_number = str(pr.number)
project_name_full = self._ci_job_name + '/PR-' + pr_number
# Retrieve console output
try:
console_output = self._jenkins.get_build_console_output(project_name_full, build_number)
except NotFoundException:
return False
# Check if build is waiting in queue (and not already running on an executor)
if 'Waiting for next available executor on' in console_output \
and 'Running on' not in console_output:
log.info('CI for PR %s: WAITING IN QUEUE', pr_number)
return True
else:
return False
def _interpret_status(self, status, pr):
"""
Verify GitHub status passed to the method.
This method verifies last commit status for given PR, calling appropriate methods
to further validate the status.
:param status: GitHub commit status
:param pr: Single PR being currently checked
:type status: github.CommitStatus.CommitStatus
:type pr: github.PullRequest.PullRequest
"""
try:
# Retrieve build number for Jenkins build related to this PR
build_number = self._retrieve_build_number(status.target_url)
# CI build finished - verify if expected output is present
finished_statuses = ['Build finished', 'This commit cannot be built', 'This commit looks good']
pending_statuses = ['This commit is being built', 'Testing in progress']
if any(phrase in status.description for phrase in finished_statuses):
self._check_finished(pr, build_number)
# CI build in progress - verify timeouts for build queue and duration
elif any(phrase in status.description for phrase in pending_statuses):
self._check_in_progress(pr, build_number)
else:
message = 'ONNX CI job for PR# {}: unrecognized status: {}'.format(pr.number, status.description)
self._queue_message(message, message_severity='error', pr=pr)
except Exception:
# Log Watchdog internal error in case any status can't be properly verified
message = 'Failed to verify status "{}" for PR# {}'.format(status.description, pr.number)
log.exception(message)
self._queue_message(message, message_severity='internal', pr=pr)
def _retrieve_build_number(self, url):
"""Retrieve Jenkins CI job build number from URL address coming from GitHub commit status.
:param url: URL address from GitHub commit status
:type url: String
:return: Returns build number
:rtype: int
"""
# Retrieve the build number from url string
match_obj = re.search('(?:/PR-[0-9]+/)([0-9]+)', url)
try:
number = int(match_obj.group(1))
return number
except Exception:
log.exception('Failed to retrieve build number from url link: %s', url)
raise
def _queue_message(self, message, message_severity='info', pr=None):
"""Add a message to message queue in communicator object.
The queued message is constructed based on message string passed as
a method argument and message header. Message header is mapped to message severity
also passed as an argument.
:param message: Message content
:param message_severity: Message severity level
:type message: String
:type message_severity: int
"""
log.info(message)
internal = False
if 'internal' in message_severity:
message_header = INTERNAL_ERROR_MESSAGE_HEADER
internal = True
elif 'error' in message_severity:
message_header = ERROR_MESSAGE_HEADER
elif 'warning' in message_severity:
message_header = WARNING_MESSAGE_HEADER
else:
message_header = INFO_MESSAGE_HEADER
# If message is related to PR attatch url
if pr:
message = message + '\n' + pr.html_url
send = message_header + '\n' + message
if self._ms_teams_enabled:
self._msteams_hook.queue_message(send)
def _check_finished(self, pr, build_number):
"""Verify if finished build output contains expected string for either fail or success.
:param pr: Single PR being currently checked
:param build_number: Jenkins CI job build number
:type pr: github.PullRequest.PullRequest
:type build_number: int
"""
pr_number = str(pr.number)
log.info('CI for PR %s: FINISHED', pr_number)
# Check if FINISH was valid FAIL / SUCCESS
project_name_full = self._ci_job_name + '/PR-' + pr_number
build_output = self._jenkins.get_build_console_output(project_name_full, build_number)
if _CI_BUILD_FAIL_MESSAGE not in build_output \
and _CI_BUILD_SUCCESS_MESSAGE not in build_output:
message = ('ONNX CI job for PR #{}: finished but no tests success or fail '
'confirmation is present in console output!'.format(pr_number))
self._queue_message(message, message_severity='error', pr=pr)
def _send_message(self, quiet=False):
"""Send messages queued in MS Teams objects to designated channel.
Queued messages are being sent as a single communication.
:param quiet: Flag for disabling sending report through communicator
:type quiet: Boolean
"""
if any(messages for messages in self._msteams_hook.messages):
try:
watchdog_build = self._jenkins.get_job_info(self._watchdog_job_name)['lastBuild']
watchdog_build_number = watchdog_build['number']
watchdog_build_link = watchdog_build['url']
except Exception:
watchdog_build_number = 'UNKNOWN'
watchdog_build_link = self._jenkins.jenkins_server
send = self._watchdog_job_name + '- build ' + str(
watchdog_build_number) + ' - ' + watchdog_build_link
if self._ms_teams_enabled:
self._msteams_hook.send_message(send, quiet=quiet)
else:
log.info('Nothing to report.')
def _check_in_progress(self, pr, build_number):
"""Check if CI build succesfully started.
Checks if build started within designated time threshold, and job is
currently running - it didn't cross the time threshold.
:param pr: Single PR being currently checked
:param build_number: Jenkins CI job build number
:type pr: github.PullRequest.PullRequest
:type build_number: int
"""
pr_number = str(pr.number)
log.info('CI for PR %s: TESTING IN PROGRESS', pr_number)
project_name_full = self._ci_job_name + '/PR-' + pr_number
build_info = self._jenkins.get_build_info(project_name_full, build_number)
build_datetime = datetime.datetime.fromtimestamp(build_info['timestamp'] / 1000.0)
build_delta = self._now_time - build_datetime
log.info('Build %s: IN PROGRESS, started: %s minutes ago', str(build_number),
str(build_delta))
# If build still waiting in queue
if build_delta > _CI_START_THRESHOLD and self._build_in_queue(pr, build_number):
message = ('ONNX CI job build #{}, for PR #{} waiting in queue after {} '
'minutes'.format(build_number, pr_number, str(build_delta.seconds / 60)))
self._queue_message(message, message_severity='warning', pr=pr)
elif build_delta > _BUILD_DURATION_THRESHOLD:
# CI job take too long, possibly froze - communicate failure
message = ('ONNX CI job build #{}, for PR #{} started,'
'but did not finish in designated time of {} '
'minutes!'.format(build_number, pr_number,
str(_BUILD_DURATION_THRESHOLD.seconds / 60)))
self._queue_message(message, message_severity='error', pr=pr)
def _update_config(self):
"""Update Watchdog config file with PRs checked in current Watchdog run, remove old entries.
:param current_prs: List of PR numbers checked during current Watchdog run
:type current_prs: list of ints
"""
# Cleanup config of old reports
log.info('Writing to config file at: {}'.format(self._config_path))
new_config = {_PR_REPORTS_CONFIG_KEY: self._current_prs}
file = open(self._config_path, 'w+')
json.dump(new_config, file)