IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Shellcheck tests for bash scripts (#2532)

Browse Source 
* Added bash scripts checks with shellcheck

* Fixed indentations

* Fixed setupvars.sh with shellcheck

* Fixed MO scripts

* Compilation with newer versions of shellcheck
This commit is contained in:
Ilya Lavrenov 2020-10-05 23:37:50 +03:00 committed by GitHub
parent 85dab59019
commit de1cc8af2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 164 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
CMakeLists.txt
View File

@ -22,8 +22,7 @@ include(features)
# include developer package
include(developer_package)
# These options are shared with 3rdparty plugins
# by means of developer package
# These options are shared with 3rdparty plugins by means of developer package
include(check_features)
include(dependencies)
@ -40,6 +39,10 @@ message (STATUS "CMAKE_BUILD_TYPE ...................... " ${CMAKE_BUILD_TYPE})
file(REMOVE "${CMAKE_BINARY_DIR}/targets_developer.cmake")
file(REMOVE "${CMAKE_BINARY_DIR}/targets.cmake")
#
# Build
#
function(build_ngraph)
function(ngraph_set option value)
if(NOT DEFINED ${option})
@ -130,7 +133,6 @@ function(openvino_developer_export_targets)
"Paths to extra Inference Engine plugins" FORCE)
endfunction()
add_subdirectory(openvino)
build_ngraph()
@ -139,7 +141,26 @@ add_subdirectory(inference-engine)
add_subdirectory(docs)
#
# Shellcheck
#
ie_shellcheck_process(DIRECTORY "${OpenVINO_MAIN_SOURCE_DIR}"
SKIP "${OpenVINO_MAIN_SOURCE_DIR}/bin"
"${OpenVINO_MAIN_SOURCE_DIR}/build"
"${IE_MAIN_SOURCE_DIR}/tests/ie_test_utils/common_test_utils/gtest"
"${IE_MAIN_SOURCE_DIR}/samples/thirdparty"
"${IE_MAIN_SOURCE_DIR}/thirdparty"
"${IE_MAIN_SOURCE_DIR}/temp"
# TODO fix and enable back:
"${OpenVINO_MAIN_SOURCE_DIR}/scripts/install_dependencies"
"${OpenVINO_MAIN_SOURCE_DIR}/scripts/demo"
"${OpenVINO_MAIN_SOURCE_DIR}/ngraph"
"${IE_MAIN_SOURCE_DIR}/scripts")
#
# cpack
#
# install setupvars

24
cmake/coverage/coverage_clean.cmake
View File

@ -3,28 +3,28 @@
#
if(NOT DEFINED IE_COVERAGE_REPORTS)
message(FATAL_ERROR "IE_COVERAGE_REPORTS variable is not defined")
return()
message(FATAL_ERROR "IE_COVERAGE_REPORTS variable is not defined")
return()
endif()
file(REMOVE_RECURSE "${IE_COVERAGE_REPORTS}")
if(NOT DEFINED IE_COVERAGE_DIRECTORY)
message(FATAL_ERROR "IE_COVERAGE_DIRECTORY variable is not defined")
return()
message(FATAL_ERROR "IE_COVERAGE_DIRECTORY variable is not defined")
return()
endif()
# remove .gcno files which are kept from the previous build
file(GLOB_RECURSE gcno_files "${IE_COVERAGE_DIRECTORY}/*.gcno")
foreach(file IN LISTS gcno_files)
string(REPLACE ".gcno" "" temp_file "${file}")
string(REGEX REPLACE "CMakeFiles/.+dir/" "" temp_file "${temp_file}")
string(REPLACE "${CMAKE_BINARY_DIRECTORY}" "${CMAKE_SOURCE_DIRECTORY}" source_file "${temp_file}")
string(REPLACE ".gcno" "" temp_file "${file}")
string(REGEX REPLACE "CMakeFiles/.+dir/" "" temp_file "${temp_file}")
string(REPLACE "${CMAKE_BINARY_DIRECTORY}" "${CMAKE_SOURCE_DIRECTORY}" source_file "${temp_file}")
if(NOT EXISTS "${source_file}")
file(REMOVE "${file}")
string(REPLACE "${CMAKE_BINARY_DIRECTORY}/" "" file "${file}")
message("Removing ${file}")
endif()
if(NOT EXISTS "${source_file}")
file(REMOVE "${file}")
string(REPLACE "${CMAKE_BINARY_DIRECTORY}/" "" file "${file}")
message("Removing ${file}")
endif()
endforeach()

15
cmake/developer_package.cmake
View File

@ -108,6 +108,7 @@ function(set_temp_directory temp_variable source_tree_dir)
endfunction()
include(coverage/coverage)
include(shellcheck/shellcheck)
# External dependencies
find_package(Threads)
@ -129,13 +130,13 @@ elseif(X86)
endif()
if(OS_FOLDER)
message ("**** OS FOLDER IS: [${OS_FOLDER}]")
if("${OS_FOLDER}" STREQUAL "ON")
message ("**** USING OS FOLDER: [${CMAKE_SYSTEM_NAME}]")
set(BIN_FOLDER "bin/${CMAKE_SYSTEM_NAME}/${ARCH_FOLDER}")
else()
set(BIN_FOLDER "bin/${OS_FOLDER}/${ARCH_FOLDER}")
endif()
message ("**** OS FOLDER IS: [${OS_FOLDER}]")
if("${OS_FOLDER}" STREQUAL "ON")
message ("**** USING OS FOLDER: [${CMAKE_SYSTEM_NAME}]")
set(BIN_FOLDER "bin/${CMAKE_SYSTEM_NAME}/${ARCH_FOLDER}")
else()
set(BIN_FOLDER "bin/${OS_FOLDER}/${ARCH_FOLDER}")
endif()
else()
set(BIN_FOLDER "bin/${ARCH_FOLDER}")
endif()

49
cmake/shellcheck/shellcheck.cmake Normal file
View File

@ -0,0 +1,49 @@
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
include(CMakeParseArguments)
find_host_program(shellcheck_PROGRAM NAMES shellcheck DOC "Path to shellcheck tool")
function(ie_shellcheck_process)
if(NOT shellcheck_PROGRAM)
message(WARNING "shellcheck tool is not found")
return()
endif()
cmake_parse_arguments(IE_SHELLCHECK "" "DIRECTORY" "SKIP" ${ARGN})
set(IE_SHELLCHECK_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/shellcheck/shellcheck_process.cmake")
file(GLOB_RECURSE scripts "${IE_SHELLCHECK_DIRECTORY}/*.sh")
foreach(script IN LISTS scripts)
# check if we need to skip scripts
unset(skip_script)
foreach(skip_directory IN LISTS IE_SHELLCHECK_SKIP)
if(script MATCHES "${skip_directory}/*")
set(skip_script ON)
endif()
endforeach()
if(skip_script)
continue()
endif()
get_filename_component(dir_name "${script}" DIRECTORY)
string(REPLACE "${IE_SHELLCHECK_DIRECTORY}" "${CMAKE_BINARY_DIR}/shellcheck" output_file ${script})
set(output_file "${output_file}.txt")
get_filename_component(script_name "${script}" NAME)
add_custom_command(OUTPUT ${output_file}
COMMAND ${CMAKE_COMMAND}
-D IE_SHELLCHECK_PROGRAM=${shellcheck_PROGRAM}
-D IE_SHELL_SCRIPT=${script}
-D IE_SHELLCHECK_OUTPUT=${output_file}
-P ${IE_SHELLCHECK_SCRIPT}
DEPENDS ${script}
COMMENT "Check script ${script_name}"
VERBATIM)
list(APPEND outputs ${output_file})
endforeach()
add_custom_target(ie_shellcheck DEPENDS ${outputs})
endfunction()

27
cmake/shellcheck/shellcheck_process.cmake Normal file
View File

@ -0,0 +1,27 @@
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
if(NOT DEFINED IE_SHELLCHECK_PROGRAM)
message(FATAL_ERROR "IE_SHELLCHECK_PROGRAM is not defined")
endif()
if(NOT DEFINED IE_SHELL_SCRIPT)
message(FATAL_ERROR "IE_SHELL_SCRIPT is not defined")
endif()
if(NOT DEFINED IE_SHELLCHECK_OUTPUT)
message(FATAL_ERROR "IE_SHELLCHECK_OUTPUT is not defined")
endif()
set(rules "SC1091,SC2236,SC2164,SC2086,SC2162,SC1090")
execute_process(COMMAND ${IE_SHELLCHECK_PROGRAM} --exclude=${rules} ${IE_SHELL_SCRIPT}
OUTPUT_VARIABLE error_message
RESULT_VARIABLE exit_code
OUTPUT_STRIP_TRAILING_WHITESPACE)
file(WRITE "${IE_SHELLCHECK_OUTPUT}" "${error_message}")
if(NOT exit_code EQUAL 0)
message(FATAL_ERROR "${error_message}")
endif()

50
inference-engine/scripts/dependencies.sh
View File

@ -3,38 +3,38 @@
# SPDX-License-Identifier: Apache-2.0
if [ "$1" = "" ]; then
dep_arch=64
else
dep_arch=$1
dep_arch=64
else
dep_arch=$1
fi
item_path=""
add_path() {
component=$1
item_path=""
component=$1
item_path=""
echo "Read file: dependencies_${dep_arch}.txt"
grep_component="\b${component}\b"
if [[ $(grep -m 1 "$grep_component" "dependencies_${dep_arch}.txt") ]];then
archive_path=$(grep -m 1 "$grep_component" "dependencies_${dep_arch}.txt" | sed -E "s/${component}=//g")
library_rpath=$(grep -m 1 "$grep_component" "ld_library_rpath_${dep_arch}.txt" | sed -E "s/${component}=//g")
filename=$(basename "$archive_path")
if [[ (! -d "$DL_SDK_TEMP/test_dependencies/$component/$filename") ||
(-d "$DL_SDK_TEMP/test_dependencies/$component/$filename" &&
! $(ls -A "$DL_SDK_TEMP/test_dependencies/$component/$filename")) ]]; then
mkdir -p "$DL_SDK_TEMP/test_dependencies/$component/$filename"
wget -q "$archive_path" -O "$DL_SDK_TEMP/test_dependencies/$filename"
if [[ $filename == *.zip ]]; then
unzip "$DL_SDK_TEMP/test_dependencies/$filename" -d "$DL_SDK_TEMP/test_dependencies/$component/$filename"
elif [[ $filename == *.7z ]]; then
7za x -y "$DL_SDK_TEMP/test_dependencies/$filename" -o "$DL_SDK_TEMP/test_dependencies/$component/$filename"
else
tar xf "$DL_SDK_TEMP/test_dependencies/$filename" -C "$DL_SDK_TEMP/test_dependencies/$component/$filename"
fi
rm "$DL_SDK_TEMP/test_dependencies/$filename"
fi
item_path=$component/$filename/$library_rpath
fi
if [[ $(grep -m 1 "$grep_component" "dependencies_${dep_arch}.txt") ]];then
archive_path=$(grep -m 1 "$grep_component" "dependencies_${dep_arch}.txt" | sed -E "s/${component}=//g")
library_rpath=$(grep -m 1 "$grep_component" "ld_library_rpath_${dep_arch}.txt" | sed -E "s/${component}=//g")
filename=$(basename "$archive_path")
if [[ (! -d "$DL_SDK_TEMP/test_dependencies/$component/$filename") ||
(-d "$DL_SDK_TEMP/test_dependencies/$component/$filename" &&
! $(ls -A "$DL_SDK_TEMP/test_dependencies/$component/$filename")) ]]; then
mkdir -p "$DL_SDK_TEMP/test_dependencies/$component/$filename"
wget -q "$archive_path" -O "$DL_SDK_TEMP/test_dependencies/$filename"
if [[ $filename == *.zip ]]; then
unzip "$DL_SDK_TEMP/test_dependencies/$filename" -d "$DL_SDK_TEMP/test_dependencies/$component/$filename"
elif [[ $filename == *.7z ]]; then
7za x -y "$DL_SDK_TEMP/test_dependencies/$filename" -o "$DL_SDK_TEMP/test_dependencies/$component/$filename"
else
tar xf "$DL_SDK_TEMP/test_dependencies/$filename" -C "$DL_SDK_TEMP/test_dependencies/$component/$filename"
fi
rm "$DL_SDK_TEMP/test_dependencies/$filename"
fi
item_path=$component/$filename/$library_rpath
fi
}
runtimes=(MKL CLDNN MYRIAD GNA DLIA OPENCV VPU_FIRMWARE_USB-MA2450 VPU_FIRMWARE_USB-MA2X8X HDDL OMP TBB AOCL_RTE LIBUSB)

4
install_dependencies.sh
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
#
params=$@
params=$1
yes_or_no() {
if [ "$params" == "-y" ]; then
@ -23,7 +23,7 @@ yes_or_no() {
if [ -f /etc/lsb-release ]; then
# Ubuntu
host_cpu=$(uname -m)
if [ $host_cpu = x86_64 ]; then
if [ "$host_cpu" = "x86_64" ]; then
x86_64_specific_packages="gcc-multilib g++-multilib"
else
x86_64_specific_packages=""

20
model-optimizer/install_prerequisites/install_prerequisites.sh
View File

@ -31,14 +31,14 @@ V_ENV=0
for ((i=1;i <= $#;i++)) {
case "${!i}" in
caffe|tf|tf2|mxnet|kaldi|onnx)
postfix="_"$1""
postfix="_$1"
;;
"venv")
V_ENV=1
;;
*)
if [[ "$1" != "" ]]; then
echo "\""${!i}"\" is unsupported parameter"
echo "\"${!i}\" is unsupported parameter"
echo $"Usage: $0 {caffe|tf|tf2|mxnet|kaldi|onnx} {venv}"
exit 1
fi
@ -80,18 +80,18 @@ fi
if [[ $V_ENV -eq 1 ]]; then
$python_binary -m venv $SCRIPTDIR/../venv
source $SCRIPTDIR/../venv/bin/activate
$SCRIPTDIR/../venv/bin/$python_binary -m pip install -r $SCRIPTDIR/../requirements${postfix}.txt
"$python_binary" -m venv "$SCRIPTDIR/../venv"
source "$SCRIPTDIR/../venv/bin/activate"
"$SCRIPTDIR/../venv/bin/$python_binary" -m pip install -r "$SCRIPTDIR/../requirements${postfix}.txt"
echo
echo "Before running the Model Optimizer, please activate virtualenv environment by running \"source ${SCRIPTDIR}/../venv/bin/activate\""
else
if [[ "$OSTYPE" == "darwin"* ]]; then
python3 -m pip install -r $SCRIPTDIR/../requirements${postfix}.txt
python3 -m pip install -r "$SCRIPTDIR/../requirements${postfix}.txt"
else
sudo -E $python_binary -m pip install -r $SCRIPTDIR/../requirements${postfix}.txt
sudo -E $python_binary -m pip install -r "$SCRIPTDIR/../requirements${postfix}.txt"
fi
echo [WARNING] All Model Optimizer dependencies are installed globally.
echo [WARNING] If you want to keep Model Optimizer in separate sandbox
echo [WARNING] run install_prerequisites.sh venv "{caffe|tf|tf2|mxnet|kaldi|onnx}"
echo "[WARNING] All Model Optimizer dependencies are installed globally."
echo "[WARNING] If you want to keep Model Optimizer in separate sandbox"
echo "[WARNING] run install_prerequisites.sh venv \"{caffe|tf|tf2|mxnet|kaldi|onnx}\""
fi

14
scripts/setupvars/setupvars.sh
View File

@ -40,9 +40,9 @@ esac
shift
done
if [ -e $INSTALLDIR/deployment_tools/inference_engine ]; then
if [ -e "$INSTALLDIR/deployment_tools/inference_engine" ]; then
export InferenceEngine_DIR=$INTEL_OPENVINO_DIR/deployment_tools/inference_engine/share
system_type=$(\ls $INTEL_OPENVINO_DIR/deployment_tools/inference_engine/lib/)
system_type=$(ls "$INTEL_OPENVINO_DIR/deployment_tools/inference_engine/lib/")
IE_PLUGINS_PATH=$INTEL_OPENVINO_DIR/deployment_tools/inference_engine/lib/$system_type
if [[ -e ${IE_PLUGINS_PATH}/arch_descriptions ]]; then
@ -59,12 +59,12 @@ if [ -e $INSTALLDIR/deployment_tools/inference_engine ]; then
HDDL_UNITE_DIR=$INSTALLDIR/deployment_tools/inference_engine/external/hddl_unite
if [ -e $HDDL_UNITE_DIR ]; then
if [ -e "$HDDL_UNITE_DIR" ]; then
export LD_LIBRARY_PATH=$HDDL_UNITE_DIR/lib:$HDDL_UNITE_DIR/thirdparty/XLink/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
fi
fi
if [ -e $INSTALLDIR/deployment_tools/ngraph ]; then
if [ -e "$INSTALLDIR/deployment_tools/ngraph" ]; then
export LD_LIBRARY_PATH=$INSTALLDIR/deployment_tools/ngraph/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
export ngraph_DIR=$INSTALLDIR/deployment_tools/ngraph/cmake
fi
@ -88,11 +88,11 @@ export PATH="$INTEL_OPENVINO_DIR/deployment_tools/model_optimizer${PATH:+:$PATH}
export PYTHONPATH="$INTEL_OPENVINO_DIR/deployment_tools/model_optimizer${PYTHONPATH:+:$PYTHONPATH}"
if [ -e $INTEL_OPENVINO_DIR/deployment_tools/open_model_zoo/tools/accuracy_checker ]; then
if [ -e "$INTEL_OPENVINO_DIR/deployment_tools/open_model_zoo/tools/accuracy_checker" ]; then
export PYTHONPATH="$INTEL_OPENVINO_DIR/deployment_tools/open_model_zoo/tools/accuracy_checker:$PYTHONPATH"
fi
if [ -e $INTEL_OPENVINO_DIR/deployment_tools/tools/post_training_optimization_toolkit ]; then
if [ -e "$INTEL_OPENVINO_DIR/deployment_tools/tools/post_training_optimization_toolkit" ]; then
export PYTHONPATH="$INTEL_OPENVINO_DIR/deployment_tools/tools/post_training_optimization_toolkit:$PYTHONPATH"
fi
@ -107,7 +107,7 @@ fi
python_bitness=$(python3 -c 'import sys; print(64 if sys.maxsize > 2**32 else 32)')
if [ "$python_bitness" != "" ] && [ "$python_bitness" != "64" ] && [ "$OS_NAME" != "Raspbian" ]; then
echo "[setupvars.sh] 64 bitness for Python" $python_version "is requred"
echo "[setupvars.sh] 64 bitness for Python $python_version is required"
fi
MINIMUM_REQUIRED_PYTHON_VERSION="3.6"