From eb775a3833dce280d8c03e0aac1c64a1a11e3d0a Mon Sep 17 00:00:00 2001 From: Alexander Zhogov Date: Thu, 18 Feb 2021 12:21:33 +0300 Subject: [PATCH] Test IncrediBuild with OpenVINO tests (#2534) * Test IncrediBuild with OpenVINO tests * Fix * Remove --gtest_print_time=1 * Move Stop IncrediBuild * Add analyze_gtest_log.py * fix * Fix * Fix * fix * Fix * Fix * CPU FuncTests Analyze IB * Fix * Remove unused pipelines * Revert "Remove unused pipelines" This reverts commit d99cdcf5971ee32e22cd32b9a6fd47815d555199. * Update to master * Exclude one test * Enable test again * Set testlevel 40 (was 128) * Try gtest-parallel and /testlevel=24 * Fix issue * Set /testlevel=8 * Set /testlevel=16 * Final fix --- .ci/azure/analyze_gtest_log.py | 53 ++++++++++++++++++++++++++++++++++ .ci/azure/windows.yml | 35 +++++++++++----------- 2 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 .ci/azure/analyze_gtest_log.py diff --git a/.ci/azure/analyze_gtest_log.py b/.ci/azure/analyze_gtest_log.py new file mode 100644 index 00000000000..c1c84d227bc --- /dev/null +++ b/.ci/azure/analyze_gtest_log.py @@ -0,0 +1,53 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +""" +Analyze GTest logs +""" + +import re +from argparse import ArgumentParser + + +def get_passed_tests(log_file_path): + """Gets passed tests with OK status""" + ok_test_line_pattern = "[ OK ] " + ok_tests = [] + with open(log_file_path) as log_file_obj: + for line in log_file_obj.readlines(): + if ok_test_line_pattern in line: + ok_tests.append(line.split(ok_test_line_pattern)[1]) + return ok_tests + + +def get_total_time(tests): + """Gets total execution time (sec)""" + re_compile_time = re.compile(r".+ \(([0-9]+) ms\)") + total_time = 0.0 + for test in tests: + re_time = re_compile_time.match(test) + if re_time: + total_time += int(re_time.group(1)) / 1000 + else: + print("No time in the test line:", test) + return total_time + + +def main(): + """The main entry point function""" + arg_parser = ArgumentParser() + arg_parser.add_argument( + "--log-file", metavar="PATH", default="gtest.log", help="Path to GTest log file" + ) + args = arg_parser.parse_args() + + passed_tests = get_passed_tests(args.log_file) + print("PASSED tests count:", len(passed_tests)) + print("Total execution time of passed tests (sec):", get_total_time(passed_tests)) + + print("\nPASSED tests:") + print("".join(sorted(passed_tests))) + + +if __name__ == "__main__": + main() diff --git a/.ci/azure/windows.yml b/.ci/azure/windows.yml index ecb5f96fc16..354151eafa7 100644 --- a/.ci/azure/windows.yml +++ b/.ci/azure/windows.yml @@ -32,9 +32,11 @@ jobs: BIN_DIR: $(REPO_DIR)\bin\intel64 MSVS_VARS_PATH: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat MSVC_COMPILER_PATH: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\MSVC\14.24.28314\bin\Hostx64\x64\cl.exe - TEST_ENV_PATH: $(REPO_DIR)\inference-engine\temp\tbb\bin;$(REPO_DIR)\inference-engine\temp\opencv_4.5.1\opencv\bin;%PATH% INSTALL_DIR: $(WORK_DIR)\install_pkg SETUPVARS: $(INSTALL_DIR)\bin\setupvars.bat + IB_DIR: C:\Program Files (x86)\IncrediBuild + IB_TESTCONSOLE: $(IB_DIR)\IBTestConsole.exe + TEST_ENV_PATH: $(REPO_DIR)\inference-engine\temp\tbb\bin;$(REPO_DIR)\inference-engine\temp\opencv_4.5.1\opencv\bin;$(IB_DIR);%PATH% steps: - script: | @@ -56,6 +58,12 @@ jobs: rd /Q /S $(BUILD_DIR) & mkdir $(BUILD_DIR) displayName: 'Make dir' + - script: | + certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat + call install_ib_console.bat + workingDirectory: $(WORK_DIR) + displayName: 'Install IncrediBuild' + - checkout: self clean: true lfs: false @@ -80,12 +88,6 @@ jobs: workingDirectory: $(WORK_DIR) displayName: 'Install dependencies' - - script: | - certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat - call install_ib_console.bat - workingDirectory: $(WORK_DIR) - displayName: 'Install IncrediBuild' - - script: | set PATH=$(WORK_DIR)\ninja-win;%PATH% call "$(MSVS_VARS_PATH)" && cmake -GNinja -DENABLE_FASTER_BUILD=ON -DENABLE_TEMPLATE_PLUGIN=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)\modules -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR) @@ -94,7 +96,7 @@ jobs: - script: | set PATH=$(WORK_DIR)\ninja-win;%PATH% - call "$(MSVS_VARS_PATH)" && "C:\Program Files (x86)\IncrediBuild\BuildConsole.exe" /COMMAND="ninja" /MaxCPUS=40 + call "$(MSVS_VARS_PATH)" && "C:\Program Files (x86)\IncrediBuild\BuildConsole.exe" /COMMAND="ninja" workingDirectory: $(BUILD_DIR) displayName: 'Build Win' @@ -113,9 +115,8 @@ jobs: - script: | set PATH=$(TEST_ENV_PATH) - $(BIN_DIR)\InferenceEngineUnitTests --gtest_print_time=1 --gtest_output=xml:TEST-InferenceEngineUnitTests.xml - displayName: 'IE UT old' - continueOnError: false + "$(IB_TESTCONSOLE)" $(BIN_DIR)\InferenceEngineUnitTests.exe --gtest_output=xml:TEST-InferenceEngineUnitTests-IB.xml + displayName: 'IE UT old - IB' - script: | set PATH=$(TEST_ENV_PATH) @@ -161,8 +162,8 @@ jobs: - script: | set PATH=$(TEST_ENV_PATH) - $(BIN_DIR)\cpuFuncTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-cpuFuncTests.xml - displayName: 'CPU FuncTests' + "$(IB_TESTCONSOLE)" $(BIN_DIR)\cpuFuncTests.exe --gtest_filter=*smoke*:-*CompareWithRefs/base_size=16_pre_nms_topn=100_post_nms_topn=100_nms_thresh=0.7_feat_stride=1_min_size=1_ratio* --gtest_output=xml:TEST-cpuFuncTests-IB.xml /testlevel=24 + displayName: 'CPU FuncTests - IB' continueOnError: false - script: | @@ -172,13 +173,13 @@ jobs: continueOnError: false # Add for gtest-parallel, it hangs now (CVS-33386) - #python $(BUILD_DIR)\gtest-parallel\gtest-parallel $(BIN_DIR)\MklDnnFunctionalTests --workers=$(WORKERS_NUMBER) --dump_json_test_results=MklDnnFunctionalTests.json --gtest_filter=*smoke* -- --gtest_print_time=1 + #python $(WORK_DIR)\gtest-parallel\gtest-parallel $(BIN_DIR)\MklDnnFunctionalTests --workers=$(WORKERS_NUMBER) --dump_json_test_results=MklDnnFunctionalTests.json --gtest_filter=*smoke* -- --gtest_print_time=1 - script: | set PATH=$(TEST_ENV_PATH) set DATA_PATH=$(MODELS_PATH) set MODELS_PATH=$(MODELS_PATH) - $(BIN_DIR)\MklDnnFunctionalTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-MklDnnFunctionalTests.xml - displayName: 'MklDnnFunctionalTests' + "$(IB_TESTCONSOLE)" $(BIN_DIR)\MklDnnFunctionalTests.exe --gtest_filter=*smoke* --gtest_output=xml:TEST-MklDnnFunctionalTests-IB.xml + displayName: 'MklDnnFunctionalTests - IB' continueOnError: false - script: | @@ -201,7 +202,7 @@ jobs: buildPlatform: 'x64' # Optional buildConfiguration: 'Windows' # Optional #publishRunAttachments: true # Optional - + - script: echo Stop IncrediBuild_Agent && net stop IncrediBuild_Agent displayName: Stop IncrediBuild continueOnError: true