Apple "cross compilation" for protoc (#12881)

thirdparty/CMakeLists.txt

@@ -169,10 +169,9 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND
         find_package(Protobuf 3.18.2 REQUIRED)
         set(Protobuf_LITE_LIBRARIES protobuf::libprotobuf-lite)
         set(Protobuf_LIBRARIES protobuf::libprotobuf)
-        set(SYSTEM_PROTOC protobuf::protoc)
-        set(PROTOC_EXECUTABLE ${SYSTEM_PROTOC})
+        set(PROTOC_EXECUTABLE protobuf::protoc)
 
-        foreach(target ${SYSTEM_PROTOC} ${Protobuf_LIBRARIES} ${Protobuf_LITE_LIBRARIES})
+        foreach(target ${PROTOC_EXECUTABLE} ${Protobuf_LIBRARIES} ${Protobuf_LITE_LIBRARIES})
             set_property(TARGET ${target} PROPERTY IMPORTED_GLOBAL ON)
         endforeach()
     else()
@@ -180,7 +179,7 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND
     endif()
 
     # forward variables used in the other places
-    set(SYSTEM_PROTOC ${SYSTEM_PROTOC} PARENT_SCOPE)
+    set(PROTOC_DEPENDENCY ${PROTOC_DEPENDENCY} PARENT_SCOPE)
     set(PROTOC_EXECUTABLE ${PROTOC_EXECUTABLE} PARENT_SCOPE)
     set(Protobuf_LIBRARIES ${Protobuf_LIBRARIES} PARENT_SCOPE)
     set(Protobuf_LITE_LIBRARIES ${Protobuf_LITE_LIBRARIES} PARENT_SCOPE)

thirdparty/onnx/CMakeLists.txt

@@ -6,8 +6,9 @@
 # Configure and install libonnx ...
 #------------------------------------------------------------------------------
 
+# TODO: enable LTO
 set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF)
-set(OV_ONNX_NAMESPACE ngraph_onnx)
+set(OV_ONNX_NAMESPACE openvino_onnx)
 
 if(NOT DEFINED ONNX_USE_MSVC_STATIC_RUNTIME)
     set(ONNX_USE_MSVC_STATIC_RUNTIME OFF)
@@ -19,13 +20,11 @@ else()
     set(ONNX_USE_LITE_PROTO_DEFAULT ON)
 endif()
 
-set(ONNX_USE_PROTOBUF_SHARED_LIBS CACHE BOOL "Use dynamic protobuf by ONNX library" FORCE)
+set(ONNX_USE_PROTOBUF_SHARED_LIBS OFF CACHE BOOL "Use dynamic protobuf by ONNX library" FORCE)
 set(ONNX_NAMESPACE ${OV_ONNX_NAMESPACE})
 set(ONNX_USE_LITE_PROTO ${ONNX_USE_LITE_PROTO_DEFAULT} CACHE BOOL "Use protobuf lite for ONNX library" FORCE)
 set(ONNX_ML ON CACHE BOOL "Use ONNX ML" FORCE)
-if(CMAKE_CROSSCOMPILING AND NOT DEFINED ONNX_CUSTOM_PROTOC_EXECUTABLE)
-    set(ONNX_CUSTOM_PROTOC_EXECUTABLE ${SYSTEM_PROTOC})
-endif()
+set(ONNX_CUSTOM_PROTOC_EXECUTABLE ${PROTOC_EXECUTABLE})
 
 # build targets
 

thirdparty/protobuf/CMakeLists.txt

@@ -25,39 +25,19 @@ set(protobuf_VERBOSE ON)
 set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build tests" FORCE)
 set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libs" FORCE)
 set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build with zlib support" FORCE)
-set(protobuf_BUILD_CONFORMANCE OFF CACHE BOOL "" FORCE)
 
-# When we build dll libraries. These flags make sure onnx and protobuf build with /MD, not /MT.
-# These two options can't be mixed, because they requires link two imcompatiable runtime.
-if(NOT DEFINED protobuf_MSVC_STATIC_RUNTIME)
-    set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link protobuf to static runtime libraries" FORCE)
+if(CMAKE_CROSSCOMPILING OR
+    (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" AND AARCH64) OR
+    (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" AND X86_64))
+    set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "Build protoc binaries" FORCE)
+else()
+    set(protobuf_BUILD_PROTOC_BINARIES ON CACHE BOOL "Build protoc binaries" FORCE)
 endif()
 
-# WA: see CVS-90227 about X86
-if(CMAKE_CROSSCOMPILING AND NOT X86)
-    find_host_program(SYSTEM_PROTOC protoc PATHS ENV PATH)
-
-    if(SYSTEM_PROTOC)
-        execute_process(
-            COMMAND ${SYSTEM_PROTOC} --version
-            OUTPUT_VARIABLE PROTOC_VERSION
-            ERROR_VARIABLE error_message
-            RESULT_VARIABLE exit_code
-            OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-        if(NOT exit_code EQUAL 0)
-            message(FATAL_ERROR "Internal error: failed to get ${SYSTEM_PROTOC} version ${error_message}")
-        endif()
-
-        string(REPLACE " " ";" PROTOC_VERSION ${PROTOC_VERSION})
-        list(GET PROTOC_VERSION -1 PROTOC_VERSION)
-
-        message("Detected system protoc version: ${PROTOC_VERSION}")
-    else()
-        message(FATAL_ERROR "System Protobuf is needed while cross-compiling")
-    endif()
-
-    set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "Build libprotoc and protoc compiler" FORCE)
+# When we build dll libraries. These flags make sure onnx and protobuf build with /MD, not /MT.
+# These two options can't be mixed, because they requires link two incompatible runtime.
+if(NOT DEFINED protobuf_MSVC_STATIC_RUNTIME)
+    set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link protobuf to static runtime libraries" FORCE)
 endif()
 
 add_subdirectory(protobuf/cmake EXCLUDE_FROM_ALL)
@@ -87,19 +67,92 @@ if(protobuf_VERSION VERSION_LESS "3.9")
     message(FATAL_ERROR "Minimum supported version of protobuf-lite library is 3.9.0 (provided ${protobuf_VERSION})")
 endif()
 
-if(CMAKE_CROSSCOMPILING AND NOT PROTOC_VERSION VERSION_EQUAL protobuf_VERSION)
-    message(WARNING "system protobuf version does not match with the compiled one, please update system protobuf or submodule")
+# build protoc separatelly for host system processor
+
+if(NOT protobuf_BUILD_PROTOC_BINARIES)
+    include(ExternalProject)
+
+    set(HOST_PROTOC_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/protobuf/cmake")
+    set(HOST_PROTOC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_build")
+    set(HOST_PROTOC_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_install")
+    set(HOST_PROTOC_PREFIX_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_root")
+
+    if(DEFINED ENV{OECORE_NATIVE_SYSROOT})
+        # need to unset several variables which can set env to cross-environment
+        foreach(var SDKTARGETSYSROOT CONFIG_SITE OECORE_NATIVE_SYSROOT OECORE_TARGET_SYSROOT
+                    OECORE_ACLOCAL_OPTS OECORE_BASELIB OECORE_TARGET_ARCH OECORE_TARGET_OS CC CXX
+                    CPP AS LD GDB STRIP RANLIB OBJCOPY OBJDUMP READELF AR NM M4 TARGET_PREFIX
+                    CONFIGURE_FLAGS CFLAGS CXXFLAGS LDFLAGS CPPFLAGS KCFLAGS OECORE_DISTRO_VERSION
+                    OECORE_SDK_VERSION ARCH CROSS_COMPILE OE_CMAKE_TOOLCHAIN_FILE OPENSSL_CONF
+                    OE_CMAKE_FIND_LIBRARY_CUSTOM_LIB_SUFFIX PKG_CONFIG_SYSROOT_DIR PKG_CONFIG_PATH)
+            list(APPEND cmake_env --unset=${var})
+        endforeach()
+
+        # filter out PATH from yocto locations
+        string(REPLACE ":" ";" custom_path "$ENV{PATH}")
+        foreach(path IN LISTS custom_path)
+            if(NOT path MATCHES "^$ENV{OECORE_NATIVE_SYSROOT}")
+                list(APPEND clean_path "${path}")
+            endif()
+        endforeach()
+
+        find_host_program(HOST_CMAKE_COMMAND
+            NAMES cmake
+            PATHS ${clean_path}
+            DOC "Host cmake"
+            REQUIRED
+            NO_DEFAULT_PATH)
+    else()
+        set(HOST_CMAKE_COMMAND "${CMAKE_COMMAND}")
+    endif()
+
+    # compile flags
+    if(CMAKE_COMPILER_IS_GNUCXX)
+        set(compile_flags "-Wno-undef -Wno-error")
+    endif()
+
+    ExternalProject_Add(host_protoc
+        SOURCE_DIR "${HOST_PROTOC_SOURCE_DIR}"
+        CONFIGURE_COMMAND
+            "${CMAKE_COMMAND}" -E env ${cmake_env}
+                "${HOST_CMAKE_COMMAND}"
+                # "-DCMAKE_GENERATOR=${CMAKE_GENERATOR}"
+                "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}"
+                "-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}"
+                "-DCMAKE_CXX_LINKER_LAUNCHER=${CMAKE_CXX_LINKER_LAUNCHER}"
+                "-DCMAKE_C_LINKER_LAUNCHER=${CMAKE_C_LINKER_LAUNCHER}"
+                "-DCMAKE_CXX_FLAGS=${compile_flags}"
+                "-DCMAKE_C_FLAGS=${compile_flags}"
+                "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW"
+                "-DCMAKE_INSTALL_PREFIX=${HOST_PROTOC_INSTALL_DIR}"
+                "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+                # protoc specific
+                "-DTHREADS_PREFER_PTHREAD_FLAG=${THREADS_PREFER_PTHREAD_FLAG}"
+                "-Dprotobuf_VERBOSE=${protobuf_VERBOSE}"
+                "-Dprotobuf_BUILD_TESTS=${protobuf_BUILD_TESTS}"
+                "-Dprotobuf_WITH_ZLIB=${protobuf_WITH_ZLIB}"
+                "-S${HOST_PROTOC_SOURCE_DIR}"
+                "-B${HOST_PROTOC_BUILD_DIR}"
+        BINARY_DIR "${HOST_PROTOC_BUILD_DIR}"
+        INSTALL_DIR "${HOST_PROTOC_INSTALL_DIR}"
+        PREFIX "${HOST_PROTOC_PREFIX_DIR}"
+        EXCLUDE_FROM_ALL ON
+    )
+
+    add_executable(protobuf::protoc IMPORTED GLOBAL)
+    set_target_properties(protobuf::protoc PROPERTIES
+        IMPORTED_LOCATION_RELEASE "${HOST_PROTOC_INSTALL_DIR}/bin/protoc")
+    add_dependencies(protobuf::protoc host_protoc)
+
+    set(PROTOC_DEPENDENCY host_protoc PARENT_SCOPE)
+    set(PROTOC_EXECUTABLE "${HOST_PROTOC_INSTALL_DIR}/bin/protoc" PARENT_SCOPE)
+else()
+    set(PROTOC_EXECUTABLE $<TARGET_FILE:protoc> PARENT_SCOPE)
+    set(PROTOC_DEPENDENCY protoc PARENT_SCOPE)
 endif()
 
 # set to parent scope
 
-if(SYSTEM_PROTOC)
-    set(SYSTEM_PROTOC ${SYSTEM_PROTOC} PARENT_SCOPE)
-    set(PROTOC_EXECUTABLE ${SYSTEM_PROTOC} PARENT_SCOPE)
-else()
-    set(PROTOC_EXECUTABLE $<TARGET_FILE:protoc> PARENT_SCOPE)
-endif()
-
 set(protobuf_VERSION ${protobuf_VERSION} PARENT_SCOPE)
 set(Protobuf_LITE_LIBRARIES libprotobuf-lite PARENT_SCOPE)
 set(Protobuf_LIBRARIES libprotobuf PARENT_SCOPE)