IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Apple "cross compilation" for protoc (#12881)

Browse Source
This commit is contained in:
Ilya Lavrenov
2022-09-03 10:50:01 +04:00
committed by GitHub
parent ac6e180bdd
commit fb8423dcee
9 changed files with 115 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
cmake/dependencies.cmake
View File

@@ -23,36 +23,6 @@ message(STATUS "MODELS_PATH=" ${MODELS_PATH})
fetch_models_and_validation_set()
get_linux_name(LINUX_OS_NAME)
if(CMAKE_CROSSCOMPILING AND CMAKE_HOST_SYSTEM_NAME MATCHES Linux AND
CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*" AND
# for x86 we have to install libc6-amd64 to enable binary compiled for x86_64
# to run with proper loader; more safer just to disable system protoc on x86
NOT X86)
set(protoc_version "3.18.2")
RESOLVE_DEPENDENCY(SYSTEM_PROTOC_ROOT
ARCHIVE_LIN "protoc-${protoc_version}-linux-x86_64.tar.gz"
TARGET_PATH "${TEMP}/protoc-${protoc_version}-linux-x86_64"
SHA256 "42fde2b6044c1f74c7e86d4e03b43aac87128ddf57ac6ed8c4eab7a1e21bbf21"
)
debug_message(STATUS "host protoc-${protoc_version} root path = " ${SYSTEM_PROTOC_ROOT})
reset_deps_cache(SYSTEM_PROTOC)
find_host_program(
SYSTEM_PROTOC
NAMES protoc
PATHS "${SYSTEM_PROTOC_ROOT}/bin"
NO_DEFAULT_PATH)
if(NOT SYSTEM_PROTOC)
message(FATAL_ERROR "[ONNX IMPORTER] Missing host protoc binary")
endif()
update_deps_cache(SYSTEM_PROTOC "${SYSTEM_PROTOC}" "Path to host protoc for ONNX Importer")
endif()
if(ENABLE_INTEL_MYRIAD)
include(${OpenVINO_SOURCE_DIR}/src/plugins/intel_myriad/myriad_dependencies.cmake)
endif()

2
cmake/developer_package/compile_flags/os_flags.cmake
View File

@@ -343,7 +343,7 @@ else()
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-dead_strip")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-dead_strip")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-dead_strip")
if (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64")
if(CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" AND AARCH64)
# Disable -mcpu=native
ie_add_compiler_flags(-Wno-error=unused-command-line-argument)
endif()

2
cmake/developer_package/frontends/frontends.cmake
View File

@@ -142,7 +142,7 @@ macro(ov_add_frontend)
add_custom_command(
OUTPUT "${OUTPUT_PB_SRC}" "${OUTPUT_PB_HEADER}"
COMMAND ${PROTOC_EXECUTABLE} ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${FILE_DIR} ${FILE_WE}.proto
DEPENDS ${PROTOC_EXECUTABLE} ${GENERATED_PROTO}
DEPENDS ${PROTOC_DEPENDENCY} ${GENERATED_PROTO}
COMMENT "Running C++ protocol buffer compiler (${PROTOC_EXECUTABLE}) on ${GENERATED_PROTO}"
VERBATIM
COMMAND_EXPAND_LISTS)

10
cmake/developer_package/target_flags.cmake
View File

@@ -35,7 +35,15 @@ if(MSVC64 OR MINGW64)
_ie_process_msvc_generator_platform(X86_64)
elseif(MINGW OR (MSVC AND NOT CMAKE_CROSSCOMPILING))
_ie_process_msvc_generator_platform(X86)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*" OR (APPLE AND CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64"))
elseif(CMAKE_OSX_ARCHITECTURES AND APPLE)
if(CMAKE_OSX_ARCHITECTURES STREQUAL "arm64")
set(AARCH64 ON)
elseif(CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64")
set(X86_64 ON)
else()
message(FATAL_ERROR "Universal binaries currently are not supported: CMAKE_OSX_ARCHITECTURES = ${CMAKE_OSX_ARCHITECTURES}")
endif()
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*")
set(X86_64 ON)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i686.*|i386.*|x86.*|amd64.*|AMD64.*")
set(X86 ON)

6
cmake/features.cmake
View File

@@ -151,10 +151,10 @@ else()
endif()
find_host_package(PythonInterp 3 QUIET)
ie_dependent_option(ENABLE_OV_ONNX_FRONTEND "Enable ONNX FrontEnd" ${PYTHONINTERP_FOUND} "protoc_available" OFF)
ie_dependent_option(ENABLE_OV_PADDLE_FRONTEND "Enable PaddlePaddle FrontEnd" ON "protoc_available" OFF)
ie_option(ENABLE_OV_ONNX_FRONTEND "Enable ONNX FrontEnd" ${PYTHONINTERP_FOUND})
ie_option(ENABLE_OV_PADDLE_FRONTEND "Enable PaddlePaddle FrontEnd" ON)
ie_option(ENABLE_OV_IR_FRONTEND "Enable IR FrontEnd" ON)
ie_dependent_option(ENABLE_OV_TF_FRONTEND "Enable TensorFlow FrontEnd" ON "protoc_available" OFF)
ie_option(ENABLE_OV_TF_FRONTEND "Enable TensorFlow FrontEnd" ON)
ie_dependent_option(ENABLE_SYSTEM_PROTOBUF "Use system protobuf" OFF
"ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_TF_FRONTEND;BUILD_SHARED_LIBS" OFF)

2
src/plugins/intel_gpu/thirdparty/CMakeLists.txt vendored
View File

@@ -34,6 +34,8 @@ if(ENABLE_ONEDNN_FOR_GPU)
"-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
"-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}"
"-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}"
"-DCMAKE_CXX_LINKER_LAUNCHER=${CMAKE_CXX_LINKER_LAUNCHER}"
"-DCMAKE_C_LINKER_LAUNCHER=${CMAKE_C_LINKER_LAUNCHER}"
"-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}"
"-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}"
"-DCMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE=${ENABLE_LTO}"

7
thirdparty/CMakeLists.txt vendored
View File

@@ -169,10 +169,9 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND
find_package(Protobuf 3.18.2 REQUIRED)
set(Protobuf_LITE_LIBRARIES protobuf::libprotobuf-lite)
set(Protobuf_LIBRARIES protobuf::libprotobuf)
set(SYSTEM_PROTOC protobuf::protoc)
set(PROTOC_EXECUTABLE ${SYSTEM_PROTOC})
set(PROTOC_EXECUTABLE protobuf::protoc)
foreach(target ${SYSTEM_PROTOC} ${Protobuf_LIBRARIES} ${Protobuf_LITE_LIBRARIES})
foreach(target ${PROTOC_EXECUTABLE} ${Protobuf_LIBRARIES} ${Protobuf_LITE_LIBRARIES})
set_property(TARGET ${target} PROPERTY IMPORTED_GLOBAL ON)
endforeach()
else()
@@ -180,7 +179,7 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND
endif()
# forward variables used in the other places
set(SYSTEM_PROTOC ${SYSTEM_PROTOC} PARENT_SCOPE)
set(PROTOC_DEPENDENCY ${PROTOC_DEPENDENCY} PARENT_SCOPE)
set(PROTOC_EXECUTABLE ${PROTOC_EXECUTABLE} PARENT_SCOPE)
set(Protobuf_LIBRARIES ${Protobuf_LIBRARIES} PARENT_SCOPE)
set(Protobuf_LITE_LIBRARIES ${Protobuf_LITE_LIBRARIES} PARENT_SCOPE)

9
thirdparty/onnx/CMakeLists.txt vendored
View File

@@ -6,8 +6,9 @@
# Configure and install libonnx ...
#------------------------------------------------------------------------------
# TODO: enable LTO
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF)
set(OV_ONNX_NAMESPACE ngraph_onnx)
set(OV_ONNX_NAMESPACE openvino_onnx)
if(NOT DEFINED ONNX_USE_MSVC_STATIC_RUNTIME)
set(ONNX_USE_MSVC_STATIC_RUNTIME OFF)
@@ -19,13 +20,11 @@ else()
set(ONNX_USE_LITE_PROTO_DEFAULT ON)
endif()
set(ONNX_USE_PROTOBUF_SHARED_LIBS CACHE BOOL "Use dynamic protobuf by ONNX library" FORCE)
set(ONNX_USE_PROTOBUF_SHARED_LIBS OFF CACHE BOOL "Use dynamic protobuf by ONNX library" FORCE)
set(ONNX_NAMESPACE ${OV_ONNX_NAMESPACE})
set(ONNX_USE_LITE_PROTO ${ONNX_USE_LITE_PROTO_DEFAULT} CACHE BOOL "Use protobuf lite for ONNX library" FORCE)
set(ONNX_ML ON CACHE BOOL "Use ONNX ML" FORCE)
if(CMAKE_CROSSCOMPILING AND NOT DEFINED ONNX_CUSTOM_PROTOC_EXECUTABLE)
set(ONNX_CUSTOM_PROTOC_EXECUTABLE ${SYSTEM_PROTOC})
endif()
set(ONNX_CUSTOM_PROTOC_EXECUTABLE ${PROTOC_EXECUTABLE})
# build targets

131
thirdparty/protobuf/CMakeLists.txt vendored
View File

@@ -25,39 +25,19 @@ set(protobuf_VERBOSE ON)
set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build tests" FORCE)
set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libs" FORCE)
set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build with zlib support" FORCE)
set(protobuf_BUILD_CONFORMANCE OFF CACHE BOOL "" FORCE)
# When we build dll libraries. These flags make sure onnx and protobuf build with /MD, not /MT.
# These two options can't be mixed, because they requires link two imcompatiable runtime.
if(NOT DEFINED protobuf_MSVC_STATIC_RUNTIME)
set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link protobuf to static runtime libraries" FORCE)
if(CMAKE_CROSSCOMPILING OR
(CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" AND AARCH64) OR
(CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" AND X86_64))
set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "Build protoc binaries" FORCE)
else()
set(protobuf_BUILD_PROTOC_BINARIES ON CACHE BOOL "Build protoc binaries" FORCE)
endif()
# WA: see CVS-90227 about X86
if(CMAKE_CROSSCOMPILING AND NOT X86)
find_host_program(SYSTEM_PROTOC protoc PATHS ENV PATH)
if(SYSTEM_PROTOC)
execute_process(
COMMAND ${SYSTEM_PROTOC} --version
OUTPUT_VARIABLE PROTOC_VERSION
ERROR_VARIABLE error_message
RESULT_VARIABLE exit_code
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(NOT exit_code EQUAL 0)
message(FATAL_ERROR "Internal error: failed to get ${SYSTEM_PROTOC} version ${error_message}")
endif()
string(REPLACE " " ";" PROTOC_VERSION ${PROTOC_VERSION})
list(GET PROTOC_VERSION -1 PROTOC_VERSION)
message("Detected system protoc version: ${PROTOC_VERSION}")
else()
message(FATAL_ERROR "System Protobuf is needed while cross-compiling")
endif()
set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "Build libprotoc and protoc compiler" FORCE)
# When we build dll libraries. These flags make sure onnx and protobuf build with /MD, not /MT.
# These two options can't be mixed, because they requires link two incompatible runtime.
if(NOT DEFINED protobuf_MSVC_STATIC_RUNTIME)
set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link protobuf to static runtime libraries" FORCE)
endif()
add_subdirectory(protobuf/cmake EXCLUDE_FROM_ALL)
@@ -87,19 +67,92 @@ if(protobuf_VERSION VERSION_LESS "3.9")
message(FATAL_ERROR "Minimum supported version of protobuf-lite library is 3.9.0 (provided ${protobuf_VERSION})")
endif()
if(CMAKE_CROSSCOMPILING AND NOT PROTOC_VERSION VERSION_EQUAL protobuf_VERSION)
message(WARNING "system protobuf version does not match with the compiled one, please update system protobuf or submodule")
# build protoc separatelly for host system processor
if(NOT protobuf_BUILD_PROTOC_BINARIES)
include(ExternalProject)
set(HOST_PROTOC_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/protobuf/cmake")
set(HOST_PROTOC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_build")
set(HOST_PROTOC_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_install")
set(HOST_PROTOC_PREFIX_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_root")
if(DEFINED ENV{OECORE_NATIVE_SYSROOT})
# need to unset several variables which can set env to cross-environment
foreach(var SDKTARGETSYSROOT CONFIG_SITE OECORE_NATIVE_SYSROOT OECORE_TARGET_SYSROOT
OECORE_ACLOCAL_OPTS OECORE_BASELIB OECORE_TARGET_ARCH OECORE_TARGET_OS CC CXX
CPP AS LD GDB STRIP RANLIB OBJCOPY OBJDUMP READELF AR NM M4 TARGET_PREFIX
CONFIGURE_FLAGS CFLAGS CXXFLAGS LDFLAGS CPPFLAGS KCFLAGS OECORE_DISTRO_VERSION
OECORE_SDK_VERSION ARCH CROSS_COMPILE OE_CMAKE_TOOLCHAIN_FILE OPENSSL_CONF
OE_CMAKE_FIND_LIBRARY_CUSTOM_LIB_SUFFIX PKG_CONFIG_SYSROOT_DIR PKG_CONFIG_PATH)
list(APPEND cmake_env --unset=${var})
endforeach()
# filter out PATH from yocto locations
string(REPLACE ":" ";" custom_path "$ENV{PATH}")
foreach(path IN LISTS custom_path)
if(NOT path MATCHES "^$ENV{OECORE_NATIVE_SYSROOT}")
list(APPEND clean_path "${path}")
endif()
endforeach()
find_host_program(HOST_CMAKE_COMMAND
NAMES cmake
PATHS ${clean_path}
DOC "Host cmake"
REQUIRED
NO_DEFAULT_PATH)
else()
set(HOST_CMAKE_COMMAND "${CMAKE_COMMAND}")
endif()
# compile flags
if(CMAKE_COMPILER_IS_GNUCXX)
set(compile_flags "-Wno-undef -Wno-error")
endif()
ExternalProject_Add(host_protoc
SOURCE_DIR "${HOST_PROTOC_SOURCE_DIR}"
CONFIGURE_COMMAND
"${CMAKE_COMMAND}" -E env ${cmake_env}
"${HOST_CMAKE_COMMAND}"
# "-DCMAKE_GENERATOR=${CMAKE_GENERATOR}"
"-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}"
"-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}"
"-DCMAKE_CXX_LINKER_LAUNCHER=${CMAKE_CXX_LINKER_LAUNCHER}"
"-DCMAKE_C_LINKER_LAUNCHER=${CMAKE_C_LINKER_LAUNCHER}"
"-DCMAKE_CXX_FLAGS=${compile_flags}"
"-DCMAKE_C_FLAGS=${compile_flags}"
"-DCMAKE_POLICY_DEFAULT_CMP0069=NEW"
"-DCMAKE_INSTALL_PREFIX=${HOST_PROTOC_INSTALL_DIR}"
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
# protoc specific
"-DTHREADS_PREFER_PTHREAD_FLAG=${THREADS_PREFER_PTHREAD_FLAG}"
"-Dprotobuf_VERBOSE=${protobuf_VERBOSE}"
"-Dprotobuf_BUILD_TESTS=${protobuf_BUILD_TESTS}"
"-Dprotobuf_WITH_ZLIB=${protobuf_WITH_ZLIB}"
"-S${HOST_PROTOC_SOURCE_DIR}"
"-B${HOST_PROTOC_BUILD_DIR}"
BINARY_DIR "${HOST_PROTOC_BUILD_DIR}"
INSTALL_DIR "${HOST_PROTOC_INSTALL_DIR}"
PREFIX "${HOST_PROTOC_PREFIX_DIR}"
EXCLUDE_FROM_ALL ON
)
add_executable(protobuf::protoc IMPORTED GLOBAL)
set_target_properties(protobuf::protoc PROPERTIES
IMPORTED_LOCATION_RELEASE "${HOST_PROTOC_INSTALL_DIR}/bin/protoc")
add_dependencies(protobuf::protoc host_protoc)
set(PROTOC_DEPENDENCY host_protoc PARENT_SCOPE)
set(PROTOC_EXECUTABLE "${HOST_PROTOC_INSTALL_DIR}/bin/protoc" PARENT_SCOPE)
else()
set(PROTOC_EXECUTABLE $<TARGET_FILE:protoc> PARENT_SCOPE)
set(PROTOC_DEPENDENCY protoc PARENT_SCOPE)
endif()
# set to parent scope
if(SYSTEM_PROTOC)
set(SYSTEM_PROTOC ${SYSTEM_PROTOC} PARENT_SCOPE)
set(PROTOC_EXECUTABLE ${SYSTEM_PROTOC} PARENT_SCOPE)
else()
set(PROTOC_EXECUTABLE $<TARGET_FILE:protoc> PARENT_SCOPE)
endif()
set(protobuf_VERSION ${protobuf_VERSION} PARENT_SCOPE)
set(Protobuf_LITE_LIBRARIES libprotobuf-lite PARENT_SCOPE)
set(Protobuf_LIBRARIES libprotobuf PARENT_SCOPE)