Merge #3307 'Travis: Rebuild third-party deps when updated'.

.ci/before_cache.sh

@@ -15,10 +15,8 @@ rm -f "${HOME}/.cache/pip/selfcheck.json"
 
 # Update the third-party dependency cache only if the build was successful.
 if [[ -f "${SUCCESS_MARKER}" ]]; then
-  if [[ ! -f "${CACHE_MARKER}" ]] || [[ "${BUILD_NVIM_DEPS}" == true ]]; then
-    echo "Updating third-party dependency cache."
-    rm -rf "${HOME}/.cache/nvim-deps"
-    mv -T "${DEPS_INSTALL_PREFIX}" "${HOME}/.cache/nvim-deps"
-    touch "${CACHE_MARKER}"
-  fi
+  rm -rf "${HOME}/.cache/nvim-deps"
+  mv -T "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps"
+  touch "${CACHE_MARKER}"
+  echo "Updated third-party dependencies (timestamp: $(stat -c '%y' "${CACHE_MARKER}"))."
 fi

.ci/common/build.sh

@@ -11,19 +11,21 @@ build_deps() {
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} ${CMAKE_FLAGS_MINGW}"
   fi
 
-  rm -rf "${DEPS_INSTALL_PREFIX}"
+  rm -rf "${DEPS_BUILD_DIR}"
 
   # If there is a valid cache and we're not forced to recompile,
   # use cached third-party dependencies.
   if [[ -f "${CACHE_MARKER}" ]] && [[ "${BUILD_NVIM_DEPS}" != true ]]; then
     echo "Using third-party dependencies from Travis's cache (last updated: $(stat -c '%y' "${CACHE_MARKER}"))."
 
-    mkdir -p "$(dirname "${DEPS_INSTALL_PREFIX}")"
-    ln -Ts "${HOME}/.cache/nvim-deps" "${DEPS_INSTALL_PREFIX}"
-    return
+     mkdir -p "$(dirname "${DEPS_BUILD_DIR}")"
+     mv -T "${HOME}/.cache/nvim-deps" "${DEPS_BUILD_DIR}"
+  else
+    mkdir -p "${DEPS_BUILD_DIR}"
   fi
 
-  mkdir -p "${DEPS_BUILD_DIR}"
+  # Even if we're using cached dependencies, run CMake and make to
+  # update CMake configuration and update to newer deps versions.
   cd "${DEPS_BUILD_DIR}"
   echo "Configuring with '${DEPS_CMAKE_FLAGS}'."
   cmake ${DEPS_CMAKE_FLAGS} "${TRAVIS_BUILD_DIR}/third-party/"
@@ -79,7 +81,7 @@ build_nvim() {
     exit 1
   fi
   asan_check "${LOG_DIR}"
-  
+
 
   cd "${TRAVIS_BUILD_DIR}"
 }

.travis.yml

@@ -3,9 +3,7 @@ language: c
 
 env:
   global:
-    # To build third-party dependencies, set this to 'true'.
-    # TODO: Change deps caching to detect updated dependencies automatically, but
-    #       still don't rebuild deps every time.
+    # To force rebuilding of third-party dependencies, set this to 'true'.
     - BUILD_NVIM_DEPS=false
     # Travis has 1.5 virtual cores according to
     # http://docs.travis-ci.com/user/speeding-up-the-build/#Paralellizing-your-build-on-one-VM
@@ -19,9 +17,9 @@ env:
     # Build directory for Neovim.
     - BUILD_DIR="$TRAVIS_BUILD_DIR/build"
     # Build directory for third-party dependencies.
-    - DEPS_BUILD_DIR="$TRAVIS_BUILD_DIR/deps-build"
-    # Directory where compiled third-party dependencies are stored.
-    - DEPS_INSTALL_PREFIX="$HOME/nvim-deps-install"
+    - DEPS_BUILD_DIR="$HOME/nvim-deps"
+    # Directory where third-party dependency sources are downloaded to.
+    - DEPS_DOWNLOAD_DIR="$TRAVIS_BUILD_DIR/deps-downloads"
     # Install directory for Neovim.
     - INSTALL_PREFIX="$HOME/nvim-install"
     # Log directory for Clang sanitizers and Valgrind.
@@ -32,8 +30,8 @@ env:
                    -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX
                    -DUSE_GCOV=ON
                    -DBUSTED_OUTPUT_TYPE=plainTerminal
-                   -DDEPS_PREFIX=$DEPS_INSTALL_PREFIX"
-    - DEPS_CMAKE_FLAGS="-DDEPS_INSTALL_DIR:PATH=$DEPS_INSTALL_PREFIX"
+                   -DDEPS_PREFIX=$DEPS_BUILD_DIR/usr"
+    - DEPS_CMAKE_FLAGS="-DDEPS_DOWNLOAD_DIR:PATH=$DEPS_DOWNLOAD_DIR"
     # Additional CMake flags for 32-bit builds.
     - CMAKE_FLAGS_32BIT="-DCMAKE_SYSTEM_LIBRARY_PATH=/lib32:/usr/lib32:/usr/local/lib32
                          -DCMAKE_IGNORE_PATH=/lib:/usr/lib:/usr/local/lib

third-party/CMakeLists.txt

@@ -22,6 +22,8 @@ option(USE_BUNDLED_MSGPACK "Use the bundled msgpack." ${USE_BUNDLED})
 option(USE_BUNDLED_LUAJIT "Use the bundled version of luajit." ${USE_BUNDLED})
 option(USE_BUNDLED_LUAROCKS "Use the bundled version of luarocks." ${USE_BUNDLED})
 
+option(USE_EXISTING_SRC_DIR "Skip download of deps sources in case of existing source directory." OFF)
+
 if(UNIX)
   find_program(MAKE_PRG NAMES gmake make)
   if(MAKE_PRG)
@@ -81,8 +83,8 @@ set(LUAJIT_SHA256 620fa4eb12375021bef6e4f237cbd2dd5d49e56beb414bee052c746beef180
 set(LUAROCKS_URL https://github.com/keplerproject/luarocks/archive/5d8a16526573b36d5b22aa74866120c998466697.tar.gz)
 set(LUAROCKS_SHA256 cae709111c5701235770047dfd7169f66b82ae1c7b9b79207f9df0afb722bfd9)
 
-set(UNIBILIUM_URL https://github.com/mauke/unibilium/archive/v1.1.4.tar.gz)
-set(UNIBILIUM_SHA256 8b8948266eb370eef8100f401d530451d627a17c068a3f85cd5d62a57517aaa7)
+set(UNIBILIUM_URL https://github.com/mauke/unibilium/archive/v1.2.0.tar.gz)
+set(UNIBILIUM_SHA256 623af1099515e673abfd3cae5f2fa808a09ca55dda1c65a7b5c9424eb304ead8)
 
 set(LIBTERMKEY_URL https://github.com/neovim/libtermkey/archive/8c0cb7108cc63218ea19aa898968eede19e19603.tar.gz)
 set(LIBTERMKEY_SHA256 21846369081e6c9a0b615f4b3889c4cb809321c5ccc6e6c1640eb138f1590072)

third-party/cmake/BuildJeMalloc.cmake

@@ -13,6 +13,7 @@ ExternalProject_Add(jemalloc
     -DURL=${JEMALLOC_URL}
     -DEXPECTED_SHA256=${JEMALLOC_SHA256}
     -DTARGET=jemalloc
+    -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR}
     -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake
   BUILD_IN_SOURCE 1
   CONFIGURE_COMMAND ${DEPS_BUILD_DIR}/src/jemalloc/configure --enable-cc-silence

third-party/cmake/BuildLibtermkey.cmake

@@ -14,6 +14,7 @@ ExternalProject_Add(libtermkey
     -DURL=${LIBTERMKEY_URL}
     -DEXPECTED_SHA256=${LIBTERMKEY_SHA256}
     -DTARGET=libtermkey
+    -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR}
     -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake
   CONFIGURE_COMMAND ""
   BUILD_IN_SOURCE 1

third-party/cmake/BuildLibuv.cmake

@@ -28,6 +28,7 @@ function(BuildLibuv)
       -DURL=${LIBUV_URL}
       -DEXPECTED_SHA256=${LIBUV_SHA256}
       -DTARGET=${_libuv_TARGET}
+      -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR}
       -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake
     CONFIGURE_COMMAND "${_libuv_CONFIGURE_COMMAND}"
     BUILD_COMMAND "${_libuv_BUILD_COMMAND}"

third-party/cmake/BuildLibvterm.cmake

@@ -13,6 +13,7 @@ ExternalProject_Add(libvterm
     -DURL=${LIBVTERM_URL}
     -DEXPECTED_SHA256=${LIBVTERM_SHA256}
     -DTARGET=libvterm
+    -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR}
     -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake
   CONFIGURE_COMMAND ""
   BUILD_IN_SOURCE 1

third-party/cmake/BuildLuajit.cmake

@@ -27,6 +27,7 @@ function(BuildLuajit)
       -DURL=${LUAJIT_URL}
       -DEXPECTED_SHA256=${LUAJIT_SHA256}
       -DTARGET=${_luajit_TARGET}
+      -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR}
       -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake
     CONFIGURE_COMMAND "${_luajit_CONFIGURE_COMMAND}"
     BUILD_IN_SOURCE 1

third-party/cmake/BuildLuarocks.cmake

@@ -34,6 +34,7 @@ function(BuildLuarocks)
       -DURL=${LUAROCKS_URL}
       -DEXPECTED_SHA256=${LUAROCKS_SHA256}
       -DTARGET=luarocks
+      -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR}
       -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake
     BUILD_IN_SOURCE 1
     CONFIGURE_COMMAND "${_luarocks_CONFIGURE_COMMAND}"

third-party/cmake/BuildMsgpack.cmake

@@ -25,6 +25,7 @@ function(BuildMsgpack)
       -DURL=${MSGPACK_URL}
       -DEXPECTED_SHA256=${MSGPACK_SHA256}
       -DTARGET=msgpack
+      -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR}
       -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake
     CONFIGURE_COMMAND "${_msgpack_CONFIGURE_COMMAND}"
     BUILD_COMMAND "${_msgpack_BUILD_COMMAND}"

third-party/cmake/BuildUnibilium.cmake

@@ -13,6 +13,7 @@ ExternalProject_Add(unibilium
     -DURL=${UNIBILIUM_URL}
     -DEXPECTED_SHA256=${UNIBILIUM_SHA256}
     -DTARGET=unibilium
+    -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR}
     -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake
   CONFIGURE_COMMAND ""
   BUILD_IN_SOURCE 1

third-party/cmake/DownloadAndExtractFile.cmake

@@ -22,12 +22,15 @@ set(SRC_DIR ${PREFIX}/src/${TARGET})
 
 # Check whether the source has been downloaded. If true, skip it.
 # Useful for external downloads like homebrew.
-if(EXISTS "${SRC_DIR}" AND IS_DIRECTORY "${SRC_DIR}")
-  file(GLOB EXISTED_FILES "${SRC_DIR}/*")
-  if(EXISTED_FILES)
-    message(STATUS "${SRC_DIR} is found and not empty, skipping download and extraction. ")
-    return()
+if(USE_EXISTING_SRC_DIR)
+  if(EXISTS "${SRC_DIR}" AND IS_DIRECTORY "${SRC_DIR}")
+    file(GLOB EXISTED_FILES "${SRC_DIR}/*")
+    if(EXISTED_FILES)
+      message(STATUS "${SRC_DIR} is found and not empty, skipping download and extraction. ")
+      return()
+    endif()
   endif()
+  message(FATAL_ERROR "USE_EXISTING_SRC_DIR set to ON, but '${SRC_DIR}' does not exist or is empty.")
 endif()
 
 # Taken from ExternalProject_Add.  Let's hope we can drop this one day when