Add profiler to document build scripts (#14029)

* Add profiler to document build scripts

* Add script name

* Use system doxygen and try to understand the build of each cmake target

* Revert doxygen

* Try to speed up documentation build

* Try to cache results

* Added cache for sphinx

* Remove profilers

* Fix install dependencies

* Restored config options

* Added ccache

* Cache python and update sphinx

* Update packaging

* Revert sphinx upgrade

* Don't need to clone recursive

* Revert "Don't need to clone recursive"

This reverts commit 7a2043df7b.

* Change cache apt action

* Updated ccache calls

* Don't use recursice init

* Update sphinx to 4.5

* Fixed indent

* Disable all custom code in theme

* Remove quite mode

* Revert "Disable all custom code in theme"

This reverts commit 096ca2f329.

* Removed dependency on python

* Disable code analyse

* Disable common functions

* Revert edit page

* Minimize the number of cycles for li

* Fixed is none

* Try to fix build

* Disable sidebar

* Revert "Disable sidebar"

This reverts commit 64303ee94f.

* Return raw html

* Removed private information

* Avoid generation multiple artifacts

* Added additional step to collect ccache

* Small change

* Fixed typo

* Added comments

.github/workflows/build_doc.yml

@@ -13,18 +13,51 @@ jobs:
       - name: Clone OpenVINO
         uses: actions/checkout@v2
         with:
-          submodules: recursive
+          submodules: true
           lfs: true
 
+      # cache from ccache
+      - name: Cache ccache
+        id: cache_restore
+        uses: actions/cache@v3
+        with:
+          path: .ccache
+          key: doc-ccache-cache
+
+      - name: Install ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: doc-ccache-cache
+          save: false
+
+      # cache from doxygen build
+      - name: Cache doxygen make
+        id: cache_doxygen_make
+        uses: actions/cache@v3
+        with:
+          path: doxygen-build
+          key: build-doxygen-cache
+
+      # cache from python packages
+      - name: Cache python requirements
+        id: cache_python_requirements
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: doc-python-requirements
+
+      - name: Install apt-get tools
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        # install doc dependencies
+        with:
+          packages: graphviz texlive liblua5.2-0
+          version: 3.0
+
       - name: Install dependencies
         run: |
           set -e
-          # install doc dependencies
-          sudo apt update
-          sudo apt --assume-yes install libusb-1.0-0-dev graphviz texlive liblua5.2-0
-          cd docs
-          python3 -m pip install -r requirements.txt --user
-          cd openvino_sphinx_theme
+          python3 -m pip install -r docs/requirements.txt --user
+          cd docs/openvino_sphinx_theme
           python3 setup.py install --user
           cd ../..
           # install doxyrest
@@ -34,25 +67,35 @@ jobs:
           # install doxygen
           mkdir doxygen
           cd doxygen
-          git clone https://github.com/doxygen/doxygen.git
-          cd doxygen
-          git checkout Release_1_9_2
-          mkdir build
-          cd build
-          cmake ..
-          cmake --build . -j`nproc`
-          sudo make install
+          git clone --depth 1 https://github.com/doxygen/doxygen.git -b Release_1_9_2
+          cmake \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -S doxygen \
+            -B ../doxygen-build
+          cmake --build ../doxygen-build -j`nproc`
+          sudo cmake --build ../doxygen-build --target install
 
       - name: CMake doc
         run: |
-          mkdir build
-          cd build
-          cmake -DENABLE_DOCS=ON -DENABLE_PYTHON=ON -DCMAKE_BUILD_TYPE=Release ..
+          cmake \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DENABLE_INTEL_MYRIAD_COMMON=OFF \
+            -DENABLE_DOCS=ON \
+            -DCMAKE_BUILD_TYPE=Release \
+            -B build
+
+      - name: Cache documentation
+        id: cache_sphinx_docs
+        uses: actions/cache@v3
+        with:
+          path: build/docs/_build/.doctrees
+          key: sphinx-docs-cache
       
       - name: Build doc
         run: |
-          cmake --build . --target sphinx_docs -j`nproc`
-        working-directory: build
+          cmake --build build --target sphinx_docs -j`nproc`
       
       - name: Archive HTML
         run: |

docs/CMakeLists.txt

@@ -30,15 +30,11 @@ function(build_docs)
 
     # Preprocessing scripts
     set(DOXY_MD_FILTER "${SCRIPTS_DIR}/doxy_md_filter.py")
-    set(PYNGRAPH_REF_SCRIPT "${SCRIPTS_DIR}/pyngraph_ref.py")
-    set(DOXY_LOG_SCRIPT "${SCRIPTS_DIR}/log.py")
-    set(PYX_FILTER "${SCRIPTS_DIR}/pyx_filter.py")
     set(PREPARE_XML_SCRIPT "${SCRIPTS_DIR}/prepare_xml.py")
     set(REMOVE_XML_SCRIPT "${SCRIPTS_DIR}/remove_xml.py")
     set(COPY_IMAGES_SCRIPT "${SCRIPTS_DIR}/copy_images.py")
     set(DOC_TEST_DIR "${SCRIPTS_DIR}/tests")
     set(DOXYGEN_MAPPING_SCRIPT "${SCRIPTS_DIR}/create_mapping.py")
-    set(DOXYGEN_MAPPING_FILE "${DOCS_BUILD_DIR}/mapping.json")
 
     # out dirs
     set(XML_OUTPUT "${DOCS_BUILD_DIR}/xml")
@@ -87,20 +83,15 @@ function(build_docs)
 
     if(GRAPH_CSV_DIR)
         set(GRAPH_CSV_DIR_OUT "${RST_OUTPUT}/csv")
-    list(APPEND commands
-        COMMAND ${CMAKE_COMMAND} -E copy_directory "${GRAPH_CSV_DIR}" "${GRAPH_CSV_DIR_OUT}"
-    )
+        list(APPEND commands
+            COMMAND ${CMAKE_COMMAND} -E copy_directory "${GRAPH_CSV_DIR}" "${GRAPH_CSV_DIR_OUT}"
+        )
     endif()
 
     list(APPEND commands
         COMMAND ${CMAKE_COMMAND} -E copy ${API_DOCS_IN}/api_reference.rst ${API_DOCS_OUT}/api_reference.rst
-    )
-
-    if(ENABLE_PYTHON)
-    list(APPEND commands
         COMMAND ${CMAKE_COMMAND} -E copy_directory ${API_DOCS_IN}/ie_python_api ${API_DOCS_OUT}/ie_python_api
-        )
-    endif()
+    )
 
     # omz doc files
     if(EXISTS "${OMZ_DOCS_DIR}")
@@ -177,7 +168,7 @@ function(build_docs)
 
     add_custom_target(sphinx_docs
                       DEPENDS doxygen_xml
-                      COMMAND sphinx-build -j auto -w ${DOCS_BUILD_DIR}/sphinx.log -b html ${RST_OUTPUT} ${SPHINX_OUTPUT}
+                      COMMAND sphinx-build -q -j auto -w ${DOCS_BUILD_DIR}/sphinx.log -b html ${RST_OUTPUT} ${SPHINX_OUTPUT}
                       WORKING_DIRECTORY ${RST_OUTPUT}
                       VERBATIM)
 

docs/conf.py

@@ -63,8 +63,7 @@ templates_path = ['_templates']
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db',
-                    '.DS_Store', 'openvino/inference-engine']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
 
 panels_add_bootstrap_css = False

docs/openvino_sphinx_theme/README.md

@@ -43,5 +43,4 @@ If you want to add version selector you must define both `current_language` and
 
 ### Maintainers
 
-* Nikolay Tyukaev <nikolay.tyukaev@intel.com>
-* Andrey Zaytsev <andrey.zaytsev@intel.com>
+* OpenVINO Documentation team

docs/openvino_sphinx_theme/openvino_sphinx_theme/__init__.py

@@ -6,7 +6,7 @@ from sphinx.errors import ExtensionError
 import jinja2
 from docutils.parsers import rst
 from pathlib import Path
-from bs4 import BeautifulSoup as bs
+from bs4 import BeautifulSoup
 from sphinx.util import logging
 from pydata_sphinx_theme import index_toctree
 from .directives.code import DoxygenSnippet
@@ -87,51 +87,6 @@ def get_theme_path():
     return theme_path
 
 
-# override pydata_sphinx_theme
-def _add_collapse_checkboxes(soup, open_first=False):
-    # based on https://github.com/pradyunsg/furo
-
-    toctree_checkbox_count = 0
-
-    for element in soup.find_all("li", recursive=True):
-        # We check all "li" elements, to add a "current-page" to the correct li.
-        classes = element.get("class", [])
-
-        # Nothing more to do, unless this has "children"
-        if not element.find("ul"):
-            continue
-
-        # Add a class to indicate that this has children.
-        element["class"] = classes + ["has-children"]
-
-        # We're gonna add a checkbox.
-        toctree_checkbox_count += 1
-        checkbox_name = f"toctree-checkbox-{toctree_checkbox_count}"
-
-        # Add the "label" for the checkbox which will get filled.
-        if soup.new_tag is None:
-            continue
-        label = soup.new_tag("label", attrs={"for": checkbox_name})
-        label.append(soup.new_tag("i", attrs={"class": "fas fa-chevron-down"}))
-        element.insert(1, label)
-
-        # Add the checkbox that's used to store expanded/collapsed state.
-        checkbox = soup.new_tag(
-            "input",
-            attrs={
-                "type": "checkbox",
-                "class": ["toctree-checkbox"],
-                "id": checkbox_name,
-                "name": checkbox_name,
-            },
-        )
-        # if this has a "current" class, be expanded by default
-        # (by checking the checkbox)
-        if "current" in classes or (open_first and toctree_checkbox_count == 1):
-            checkbox.attrs["checked"] = ""
-        element.insert(1, checkbox)
-
-
 def add_toctree_functions(app, pagename, templatename, context, doctree):
 
     # override pydata_sphinx_theme
@@ -173,42 +128,73 @@ def add_toctree_functions(app, pagename, templatename, context, doctree):
             # select the "active" subset of the navigation tree for the sidebar
             toc_sphinx = index_toctree(app, pagename, startdepth, **kwargs)
 
-        soup = bs(toc_sphinx, "html.parser")
+        soup = BeautifulSoup(toc_sphinx, "html.parser")
 
-        # pair "current" with "active" since that's what we use w/ bootstrap
-        for li in soup("li", {"class": "current"}):
-            li["class"].append("active")
-
-        # Remove navbar/sidebar links to sub-headers on the page
-        for li in soup.select("li"):
-            # Remove
-            if li.find("a"):
-                href = li.find("a")["href"]
-                if "#" in href and href != "#":
-                    li.decompose()
-
-        if kind == "navbar":
-            # Add CSS for bootstrap
-            for li in soup("li"):
-                li["class"].append("nav-item")
-                li.find("a")["class"].append("nav-link")
-            # only select li items (not eg captions)
-            out = "\n".join([ii.prettify() for ii in soup.find_all("li")])
-
-        elif kind == "sidebar":
+        if kind == "sidebar":
             # Add bootstrap classes for first `ul` items
             for ul in soup("ul", recursive=False):
                 ul.attrs["class"] = ul.attrs.get("class", []) + ["nav", "bd-sidenav"]
 
-            # Add icons and labels for collapsible nested sections
-            _add_collapse_checkboxes(soup, open_first=open_first)
+        toctree_checkbox_count = 0
 
-            out = soup.prettify()
+        for li in soup.find_all("li"):
+            # pair "current" with "active" since that's what we use w/ bootstrap
+            if "current" in li["class"]:
+                li["class"].append("active")
 
-        elif kind == "raw":
-            out = soup
+            # Remove navbar/sidebar links to sub-headers on the page
+            if li.find("a"):
+                href = li.find("a")["href"]
+                if "#" in href and href != "#":
+                    li.decompose()
+                    continue
 
-        return out
+            if kind == "navbar":
+                li["class"].append("nav-item")
+                li.find("a")["class"].append("nav-link")
+                # only select li items (not eg captions)
+                # out = "\n".join([ii.prettify() for ii in soup.find_all("li")])
+            elif kind == "sidebar":
+                if li is None:
+                    continue
+                # We check all "li" elements, to add a "current-page" to the correct li.
+                classes = li.get("class", [])
+
+                # Nothing more to do, unless this has "children"
+                if not li.find("ul"):
+                    continue
+
+                # Add a class to indicate that this has children.
+                li["class"] = classes + ["has-children"]
+
+                # We're gonna add a checkbox.
+                toctree_checkbox_count += 1
+                checkbox_name = f"toctree-checkbox-{toctree_checkbox_count}"
+
+                # Add the "label" for the checkbox which will get filled.
+                if soup.new_tag is None:
+                    continue
+                label = soup.new_tag("label", attrs={"for": checkbox_name})
+                label.append(soup.new_tag("i", attrs={"class": "fas fa-chevron-down"}))
+                li.insert(1, label)
+
+                # Add the checkbox that's used to store expanded/collapsed state.
+                checkbox = soup.new_tag(
+                    "input",
+                    attrs={
+                        "type": "checkbox",
+                        "class": ["toctree-checkbox"],
+                        "id": checkbox_name,
+                        "name": checkbox_name,
+                    },
+                )
+                # if this has a "current" class, be expanded by default
+                # (by checking the checkbox)
+                if "current" in classes or (open_first and toctree_checkbox_count == 1):
+                    checkbox.attrs["checked"] = ""
+                li.insert(1, checkbox)
+
+        return soup
 
     context["generate_sidebar_nav"] = generate_sidebar_nav
 

docs/openvino_sphinx_theme/setup.py

@@ -7,8 +7,7 @@ setup(
     name='openvino-sphinx-theme',
     version='0.0.1',
     packages=['openvino_sphinx_theme'],
-    maintainer='Nikolay Tyukaev',
-    maintainer_email='nikolay.tyukaev@intel.com',
+    maintainer='OpenVINO Documentation Team',
     include_package_data=True,
     entry_points={"sphinx.html_themes": ["openvino_sphinx_theme = openvino_sphinx_theme"]},
     install_requires=['pydata_sphinx_theme', 'sphinx_inline_tabs'],

docs/requirements.txt

@@ -31,17 +31,17 @@ requests==2.25.1
 six==1.15.0
 snowballstemmer==2.1.0
 soupsieve==2.2.1
-Sphinx==3.2.1
+sphinx==4.5.0
 sphinx-copybutton==0.3.3
 sphinx-inline-tabs==2021.8.17b10
 sphinx-panels==0.6.0
 sphinx-sitemap==2.2.0
 sphinxcontrib-applehelp==1.0.2
 sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-htmlhelp==2.0.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.4
+sphinxcontrib-serializinghtml==1.1.5
 toml==0.10.2
 urllib3==1.26.5
 zipp==3.4.1

docs/scripts/remove_xml.py

@@ -4,7 +4,6 @@
 import os
 import shutil
 
-
 def remove_xml_dir(path):
     """
     Remove doxygen xml folder