cmake build all docs (#3539)

* cmake build all docs

* update doxy log parser

* update build_main_layout.py

docs/CMakeLists.txt

@@ -40,6 +40,11 @@ if(NOT ENABLE_DOCKER)
     endforeach()
 endif()
 
+set(OMZ_DOCS_DIR "" CACHE PATH "Path to open_model_zoo documentation")
+set(WORKBENCH_DOCS_DIR "" CACHE PATH "Path to workbench documentation")
+set(POT_DOCS_DIR "" CACHE PATH "Path to post-training-compression-tool documentation")
+set(GST_DOCS_DIR "" CACHE PATH "Path to gst-video-analytics documentation")
+
 function(build_docs)
     find_package(Doxygen REQUIRED dot)
     find_package(Python3 COMPONENTS Interpreter)
@@ -53,6 +58,16 @@ function(build_docs)
         message(FATAL_ERROR "Python3 is required to build the documentation")
     endif()
 
+    execute_process(
+        COMMAND ${Python3_EXECUTABLE} -m pip show lxml
+        RESULT_VARIABLE PIP_EXIT_CODE
+        OUTPUT_QUIET
+    )
+
+    if (NOT ${PIP_EXIT_CODE} EQUAL 0)
+        message(FATAL_ERROR "lxml package is not installed. Please use \"pip install lxml\".")
+    endif()
+
     if(NOT LATEX_FOUND)
         message(FATAL_ERROR "LATEX is required to build the documentation")
     endif()
@@ -70,20 +85,10 @@ function(build_docs)
 
     # Preprocessing scripts
     set(DOXY_MD_FILTER "${DOXYGEN_DIR}/doxy_md_filter.py")
+    set(DOXY_LAYOUT_SCRIPT "${DOXYGEN_DIR}/build_main_layout.py")
     set(DOXY_LOG_SCRIPT "${DOXYGEN_DIR}/log.py")
     set(PYX_FILTER "${DOXYGEN_DIR}/pyx_filter.py")
 
-    file(GLOB_RECURSE doc_source_files
-        LIST_DIRECTORIES true RELATIVE ${OpenVINO_MAIN_SOURCE_DIR}
-        "${OpenVINO_MAIN_SOURCE_DIR}/docs/*.md"
-        "${OpenVINO_MAIN_SOURCE_DIR}/docs/*.png"
-        "${OpenVINO_MAIN_SOURCE_DIR}/docs/*.gif"
-        "${OpenVINO_MAIN_SOURCE_DIR}/docs/*.jpg"
-        "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.md"
-        "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.png"
-        "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.gif"
-        "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.jpg")
-
     configure_file(${PYTHON_API_IN} ${PYTHON_API_OUT} @ONLY)
 
     set(NGRAPH_CPP_CONFIG_SOURCE "${DOXYGEN_DIR}/ngraph_cpp_api.config")
@@ -103,6 +108,7 @@ function(build_docs)
     set(NGRAPH_CPP_LAYOUT_SOURCE "${DOXYGEN_DIR}/ngraph_cpp_api.xml")
     set(NGRAPH_PY_LAYOUT_SOURCE "${DOXYGEN_DIR}/ngraph_py_api.xml")
     set(IE_LAYOUT_SOURCE "${DOXYGEN_DIR}/ie_docs.xml")
+    set(OPENVINO_LAYOUT_SOURCE "${DOXYGEN_DIR}/openvino_docs.xml")
     set(C_LAYOUT_SOURCE "${DOXYGEN_DIR}/ie_c_api.xml")
     set(PY_LAYOUT_SOURCE "${DOXYGEN_DIR}/ie_py_api.xml")
     set(PLUGIN_LAYOUT_SOURCE "${DOXYGEN_DIR}/ie_plugin_api.xml")
@@ -110,6 +116,7 @@ function(build_docs)
     set(NGRAPH_CPP_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ngraph_cpp_api.xml")
     set(NGRAPH_PY_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ngraph_py_api.xml")
     set(IE_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ie_docs.xml")
+    set(OPENVINO_LAYOUT_BUILD "${DOCS_BUILD_DIR}/openvino_docs.xml")
     set(C_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ie_c_api.xml")
     set(PY_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ie_py_api.xml")
     set(PLUGIN_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ie_plugin_api.xml")
@@ -118,6 +125,7 @@ function(build_docs)
     configure_file(${NGRAPH_CPP_LAYOUT_SOURCE} ${NGRAPH_CPP_LAYOUT_BUILD} @ONLY)
     configure_file(${NGRAPH_PY_LAYOUT_SOURCE} ${NGRAPH_PY_LAYOUT_BUILD} @ONLY)
     configure_file(${IE_LAYOUT_SOURCE} ${IE_LAYOUT_BUILD} @ONLY)
+    configure_file(${OPENVINO_LAYOUT_SOURCE} ${OPENVINO_LAYOUT_BUILD} @ONLY)
     configure_file(${C_LAYOUT_SOURCE} ${C_LAYOUT_BUILD} @ONLY)
     configure_file(${PY_LAYOUT_SOURCE} ${PY_LAYOUT_BUILD} @ONLY)
     configure_file(${PLUGIN_LAYOUT_SOURCE} ${PLUGIN_LAYOUT_BUILD} @ONLY)
@@ -175,14 +183,98 @@ function(build_docs)
                       COMMENT "Pre-process docs"
                       VERBATIM)
 
-    foreach(source_file ${doc_source_files})
+    # ovino doc files
+    file(GLOB_RECURSE ovino_doc_files
+        LIST_DIRECTORIES true RELATIVE ${OpenVINO_MAIN_SOURCE_DIR}
+        "${OpenVINO_MAIN_SOURCE_DIR}/docs/*.md"
+        "${OpenVINO_MAIN_SOURCE_DIR}/docs/*.png"
+        "${OpenVINO_MAIN_SOURCE_DIR}/docs/*.gif"
+        "${OpenVINO_MAIN_SOURCE_DIR}/docs/*.jpg"
+        "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.md"
+        "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.png"
+        "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.gif"
+        "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.jpg")
+
+    foreach(source_file ${ovino_doc_files})
         list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
-            "${OpenVINO_MAIN_SOURCE_DIR}/${source_file}" "${DOCS_BUILD_DIR}/${source_file}")
+            "${OpenVINO_MAIN_SOURCE_DIR}/${source_file}" "${DOCS_BUILD_DIR}/openvino/${source_file}")
     endforeach()
 
+    # omz doc files
+    if(EXISTS "${OMZ_DOCS_DIR}")
+        get_filename_component(OMZ_DOCS_DIR "${OMZ_DOCS_DIR}" ABSOLUTE)
+
+        file(GLOB_RECURSE omz_doc_files
+            LIST_DIRECTORIES true RELATIVE ${OMZ_DOCS_DIR}
+            "${OMZ_DOCS_DIR}/*.md"
+            "${OMZ_DOCS_DIR}/*.png"
+            "${OMZ_DOCS_DIR}/*.gif"
+            "${OMZ_DOCS_DIR}/*.jpg")
+
+        foreach(source_file ${omz_doc_files})
+            list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
+                "${OMZ_DOCS_DIR}/${source_file}" "${DOCS_BUILD_DIR}/omz/${source_file}")
+        endforeach()
+        configure_file("${OMZ_DOCS_DIR}/omz_docs.xml" "${DOCS_BUILD_DIR}/omz_docs.xml" @ONLY)
+    endif()
+
+    # workbench doc files
+    if(EXISTS "${WORKBENCH_DOCS_DIR}")
+        get_filename_component(WORKBENCH_DOCS_DIR "${WORKBENCH_DOCS_DIR}" ABSOLUTE)
+
+        file(GLOB_RECURSE workbench_doc_files
+            LIST_DIRECTORIES true RELATIVE ${WORKBENCH_DOCS_DIR}
+            "${WORKBENCH_DOCS_DIR}/*.md"
+            "${WORKBENCH_DOCS_DIR}/*.png"
+            "${WORKBENCH_DOCS_DIR}/*.gif"
+            "${WORKBENCH_DOCS_DIR}/*.jpg")
+
+        foreach(source_file ${workbench_doc_files})
+            list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
+                "${WORKBENCH_DOCS_DIR}/${source_file}" "${DOCS_BUILD_DIR}/workbench/${source_file}")
+        endforeach()
+        configure_file("${WORKBENCH_DOCS_DIR}/docs/Workbench_DG/workbench_docs.xml" "${DOCS_BUILD_DIR}/workbench_docs.xml" @ONLY)
+    endif()
+
+    # pot doc files
+    if(EXISTS "${POT_DOCS_DIR}")
+        get_filename_component(POT_DOCS_DIR "${POT_DOCS_DIR}" ABSOLUTE)
+
+        file(GLOB_RECURSE pot_doc_files
+            LIST_DIRECTORIES true RELATIVE ${POT_DOCS_DIR}
+            "${POT_DOCS_DIR}/*.md"
+            "${POT_DOCS_DIR}/*.png"
+            "${POT_DOCS_DIR}/*.gif"
+            "${POT_DOCS_DIR}/*.jpg")
+
+        foreach(source_file ${pot_doc_files})
+            list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
+                "${POT_DOCS_DIR}/${source_file}" "${DOCS_BUILD_DIR}/pot/${source_file}")
+        endforeach()
+        configure_file("${POT_DOCS_DIR}/docs/pot_docs.xml" "${DOCS_BUILD_DIR}/pot_docs.xml" @ONLY)
+    endif()
+
+    # gst doc files
+    if(EXISTS "${GST_DOCS_DIR}")
+        get_filename_component(GST_DOCS_DIR "${GST_DOCS_DIR}" ABSOLUTE)
+
+        file(GLOB_RECURSE gst_doc_files
+            LIST_DIRECTORIES true RELATIVE ${GST_DOCS_DIR}
+            "${GST_DOCS_DIR}/*.md"
+            "${GST_DOCS_DIR}/*.png"
+            "${GST_DOCS_DIR}/*.gif"
+            "${GST_DOCS_DIR}/*.jpg")
+
+        foreach(source_file ${gst_doc_files})
+            list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
+                "${GST_DOCS_DIR}/${source_file}" "${DOCS_BUILD_DIR}/gst/${source_file}")
+        endforeach()
+    endif()
+
     add_custom_command(TARGET preprocess_docs
                        PRE_BUILD
                        ${commands}
+                       COMMAND ${Python3_EXECUTABLE} ${DOXY_LAYOUT_SCRIPT} --openvino ${OPENVINO_LAYOUT_BUILD}
                        COMMAND ${Python3_EXECUTABLE} ${DOXY_MD_FILTER} ${DOCS_BUILD_DIR}
                        COMMENT "Pre-process markdown and image links")
 
@@ -196,8 +288,11 @@ function(build_docs)
 
     add_custom_command(TARGET ie_docs
                        POST_BUILD
-                       COMMAND ${Python3_EXECUTABLE} ${DOXY_LOG_SCRIPT} --log ${DOCS_BUILD_DIR}/ie_docs.log
+                       COMMAND ${Python3_EXECUTABLE} ${DOXY_LOG_SCRIPT} --log "${DOCS_BUILD_DIR}/ie_docs.log"
-                                                       --exclude-links ".*?(omz_|pot_|gst_|workbench_).*?"
+                                                                        --include_omz $<BOOL:${OMZ_DOCS_DIR}>
+                                                                        --include_wb $<BOOL:${WORKBENCH_DOCS_DIR}>
+                                                                        --include_pot $<BOOL:${POT_DOCS_DIR}>
+                                                                        --include_gst $<BOOL:${GST_DOCS_DIR}>
                        COMMENT "Parse doxygen log to find errors."
                        VERBATIM
                        )

docs/doxygen/build_main_layout.py

@@ -0,0 +1,23 @@
+import argparse
+import os
+from lxml import etree
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--openvino', type=str, required=True, default=None, help='openvino_docs.xml')
+    args = parser.parse_args()
+    result = build_layout(args.openvino)
+    with open(args.openvino, 'wb') as f:
+        f.write(result)
+
+
+def build_layout(openvino):
+    ns = {"xi": "http://www.w3.org/2001/XInclude"}
+    root = etree.parse(openvino)
+    root.xinclude()
+    return etree.tostring(root, pretty_print=True)
+
+
+if __name__ == '__main__':
+    main()

docs/doxygen/ie_docs.config

@@ -735,7 +735,7 @@ FILE_VERSION_FILTER    =
 # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
 # tag is left empty.
 
-LAYOUT_FILE            = "@IE_LAYOUT_BUILD@"
+LAYOUT_FILE            = "@OPENVINO_LAYOUT_BUILD@"
 
 # The CITE_BIB_FILES tag can be used to specify one or more bib files containing
 # the reference definitions. This must be a list of .bib files. The .bib

docs/doxygen/log.py

@@ -10,7 +10,14 @@ def parse_arguments():
                         help='Path to doxygen ignore list')
     parser.add_argument('--strip', type=str, required=False, default=os.path.abspath('../../'),
                         help='Strip from warning paths')
-    parser.add_argument('--exclude-links', nargs='+', type=str, required=False, default=[], help='Markdown links to be excluded')
+    parser.add_argument('--include_omz', type=bool, required=False, default=False,
+                        help='Include link check for omz docs')
+    parser.add_argument('--include_wb', type=bool, required=False, default=False,
+                        help='Include link check for workbench docs')
+    parser.add_argument('--include_pot', type=bool, required=False, default=False,
+                        help='Include link check for pot docs')
+    parser.add_argument('--include_gst', type=bool, required=False, default=False,
+                        help='Include link check for gst docs')
     return parser.parse_args()
 
 
@@ -37,8 +44,21 @@ def is_excluded_link(warning, exclude_links):
     return False
 
 
-def parse(log, ignore_list, strip, exclude_links):
+def parse(log, ignore_list, strip, include_omz=False, include_wb=False, include_pot=False, include_gst=False):
     found_errors = []
+    exclude_links = {'omz': r'.*?omz_.*?', 'wb': r'.*?workbench_.*?',
+                     'pot': r'.*?pot_.*?', 'gst': r'.*?gst_.*?'}
+    if include_omz:
+        del exclude_links['omz']
+    if include_wb:
+        del exclude_links['wb']
+    if include_pot:
+        del exclude_links['pot']
+    if include_gst:
+        del exclude_links['gst']
+
+    exclude_links = exclude_links.values()
+
     with open(ignore_list, 'r') as f:
         ignore_list = f.read().splitlines()
     with open(log, 'r') as f:
@@ -61,7 +81,13 @@ def parse(log, ignore_list, strip, exclude_links):
 
 def main():
     args = parse_arguments()
-    parse(args.log,  args.ignore_list, args.strip, args.exclude_links)
+    parse(args.log,
+          args.ignore_list,
+          args.strip,
+          include_omz=args.include_omz,
+          include_wb=args.include_wb,
+          include_pot=args.include_pot,
+          include_gst=args.include_gst)
 
 
 if __name__ == '__main__':

docs/doxygen/openvino_docs.xml

@@ -65,11 +65,19 @@
         <!-- DOCUMENTATION category -->
         <tab type="usergroup" title="DOCUMENTATION"><!--automatically generated-->
             <!-- DLDT Documentation-->
-            <xi:include href="ie_docs.xml" xpointer="xpointer(//tab[@id='converting_and_preparing_models'])"/>
+            <xi:include href="ie_docs.xml" xpointer="xpointer(//tab[@id='converting_and_preparing_models'])">
-            <xi:include href="ie_docs.xml" xpointer="xpointer(//tab[@id='intermediate_representaton_and_operations_sets'])"/>
+                <xi:fallback/>
-            <xi:include href="ie_docs.xml" xpointer="xpointer(//tab[@id='deploying_inference'])"/>
+            </xi:include>
+            <xi:include href="ie_docs.xml" xpointer="xpointer(//tab[@id='intermediate_representaton_and_operations_sets'])">
+                <xi:fallback/>
+            </xi:include>
+            <xi:include href="ie_docs.xml" xpointer="xpointer(//tab[@id='deploying_inference'])">
+                <xi:fallback/>
+            </xi:include>
             <!-- Workbench -->
-            <xi:include href="workbench_docs.xml" xpointer="xpointer(//tab[@id='deep_learning_workbench'])"/>
+            <xi:include href="workbench_docs.xml" xpointer="xpointer(//tab[@id='deep_learning_workbench'])">
+                <xi:fallback/>
+            </xi:include>
             <!-- Tuning for Performance -->
             <tab type="usergroup" title="Tuning for Performance">
                 <!-- Performance Benchmarks -->
@@ -80,10 +88,14 @@
                 </tab>
                 <tab type="user" title="Performance Optimization Guide" url="@ref openvino_docs_optimization_guide_dldt_optimization_guide"/>
                 <!-- POT DevGuide -->
-                <xi:include href="pot_docs.xml" xpointer="xpointer(//tab[@id='pot'])"/>
+                <xi:include href="pot_docs.xml" xpointer="xpointer(//tab[@id='pot'])">
+                    <xi:fallback/>
+                </xi:include>
                 <!-- Tuning Utilities -->
                 <tab type="usergroup" title="Tuning Utilities" url="">
-                    <xi:include href="omz_docs.xml" xpointer="xpointer(//tab[@id='accuracy_checker'])"/>
+                    <xi:include href="omz_docs.xml" xpointer="xpointer(//tab[@id='accuracy_checker'])">
+                        <xi:fallback/>
+                    </xi:include>
                     <tab type="user" title="Using Cross Check Tool for Per-Layer Comparison Between Plugins" url="@ref openvino_inference_engine_tools_cross_check_tool_README"/>
                 </tab>
                 <tab type="user" title="Case Studies" url="https://www.intel.com/openvino-success-stories"/>
@@ -108,8 +120,12 @@
         <!-- RESOURCES category -->
         <tab type="usergroup" title="RESOURCES">
             <!-- Models and Demos Documentation-->
-            <xi:include href="omz_docs.xml" xpointer="xpointer(//tab[@id='trained_models'])"/>
+            <xi:include href="omz_docs.xml" xpointer="xpointer(//tab[@id='trained_models'])">
-            <xi:include href="omz_docs.xml" xpointer="xpointer(//tab[@id='application_demos'])"/>
+                <xi:fallback/>
+            </xi:include>
+            <xi:include href="omz_docs.xml" xpointer="xpointer(//tab[@id='application_demos'])">
+                <xi:fallback/>
+            </xi:include>
             <!-- IE Code Samples -->
             <tab type="usergroup" title="Inference Engine Code Samples" url="@ref openvino_docs_IE_DG_Samples_Overview">
                 <tab type="user" title="Image Classification C++ Sample Async" url="@ref openvino_inference_engine_samples_classification_sample_async_README"/>