From 39b3eb34c24167de057c2a408e80ffa0eb281028 Mon Sep 17 00:00:00 2001 From: James McCoy Date: Sat, 14 Nov 2020 17:23:23 -0500 Subject: [PATCH 1/5] ci: Install gem binaries to ~/.local/bin That directory is already part of `$PATH`, so let's reuse it instead of trying to figure what version-specific path binaries gem normally uses. --- .travis.yml | 1 + ci/install.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 715109f71f..aad9b38934 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ env: # 4. Undo changes to .travis.yml - secure: hd0qn2u8ABbJg5Bx4pBRcUQbKYFmcSHoecyHIPTCnGJT+NI41Bvm/IkN/N5DhBF+LbD3Q2nmR/dzI5H/dqS7RxMFvEx1DuFLendFHHX3MYf0AuKpXYY3gwgMTmqx8p/v6srlU7RBGWNGzHCWqksAem+EIWCe3I7WvfdKo1/DV/Y= + - PATH="$HOME/.local/bin:$PATH" # Set "false" to force rebuild of third-party dependencies. - CACHE_ENABLE=true # Build directory for Neovim. diff --git a/ci/install.sh b/ci/install.sh index cd0d744361..ebbd820d9f 100755 --- a/ci/install.sh +++ b/ci/install.sh @@ -19,7 +19,7 @@ echo "Install neovim module for Python 2." CC=cc python2 -m pip -q install --user --upgrade pynvim echo "Install neovim RubyGem." -gem install --no-document --user-install --pre neovim +gem install --no-document --bindir "$HOME/.local/bin" --user-install --pre neovim echo "Install neovim npm package" source ~/.nvm/nvm.sh From 5271f1e2c9d6df15244cb4f7d8b63d88abc6717b Mon Sep 17 00:00:00 2001 From: James McCoy Date: Sat, 14 Nov 2020 20:21:26 -0500 Subject: [PATCH 2/5] GHA: Move env var setup into env.sh to keep it DRY --- .github/workflows/env.sh | 58 +++++++++++++++++++++++++++++++++++++ .github/workflows/linux.yml | 56 ++--------------------------------- 2 files changed, 60 insertions(+), 54 deletions(-) create mode 100755 .github/workflows/env.sh diff --git a/.github/workflows/env.sh b/.github/workflows/env.sh new file mode 100755 index 0000000000..8e005c7266 --- /dev/null +++ b/.github/workflows/env.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -e -u + +FLAVOR=$1 + +cat <> "$GITHUB_PATH" +$HOME/.local/bin +EOF + +cat <> "$GITHUB_ENV" +CACHE_ENABLE=true +CI_TARGET=tests +CI_BUILD_DIR=$GITHUB_WORKSPACE +BUILD_DIR=$GITHUB_WORKSPACE/build +DEPS_BUILD_DIR=$HOME/nvim-deps +INSTALL_PREFIX=$HOME/nvim-install +LOG_DIR=$GITHUB_WORKSPACE/build/log +NVIM_LOG_FILE=$GITHUB_WORKSPACE/build/.nvimlog +VALGRIND_LOG=$GITHUB_WORKSPACE/build/log/valgrind-%p.log +CACHE_NVIM_DEPS_DIR=$HOME/.cache/nvim-deps +CACHE_MARKER=$HOME/.cache/nvim-deps/.ci_cache_marker +CCACHE_BASEDIR=$GITHUB_WORKSPACE +DEPS_CMAKE_FLAGS=-DUSE_BUNDLED_GPERF=OFF +FUNCTIONALTEST=functionaltest +CCACHE_COMPRESS=1 +CCACHE_SLOPPINESS=time_macros,file_macro +CCACHE_CPP2=1 +EOF + +BUILD_FLAGS="CMAKE_FLAGS=-DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX:PATH=$HOME/nvim-install -DBUSTED_OUTPUT_TYPE=nvim -DDEPS_PREFIX=$HOME/nvim-deps/usr -DMIN_LOG_LEVEL=3" + +case "$FLAVOR" in + asan) + BUILD_FLAGS="$BUILD_FLAGS -DPREFER_LUA=ON" + cat <> "$GITHUB_ENV" +CLANG_SANITIZER=ASAN_UBSAN +SYMBOLIZER=asan_symbolize-11 +ASAN_OPTIONS=detect_leaks=1:check_initialization_order=1:log_path=$GITHUB_WORKSPACE/build/log/asan +UBSAN_OPTIONS=print_stacktrace=1 log_path=$GITHUB_WORKSPACE/build/log/ubsan +EOF + ;; + tsan) + cat <> "$GITHUB_ENV" +TSAN_OPTIONS=log_path=$GITHUB_WORKSPACE/build/log/tsan +EOF + ;; + lint) + cat <> "$GITHUB_ENV" +CI_TARGET=lint +EOF + ;; + *) + ;; +esac + +cat <> "$GITHUB_ENV" +$BUILD_FLAGS +EOF diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f72500efe7..f8e6556b91 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,17 +1,5 @@ name: Linux CI on: [push, pull_request] -env: - # Set "false" to force rebuild of third-party dependencies. - CACHE_ENABLE: true - DEPS_CMAKE_FLAGS: "-DUSE_BUNDLED_GPERF=OFF" - # default target name for functional tests - FUNCTIONALTEST: functionaltest - CI_TARGET: tests - # Environment variables for ccache - CCACHE_COMPRESS: 1 - CCACHE_SLOPPINESS: "time_macros,file_macro" - # Default since 3.3; required with newer gcc/clang. - CCACHE_CPP2: 1 jobs: ASAN: @@ -22,23 +10,7 @@ jobs: - uses: actions/checkout@v2 - name: Setup commom environment variables - run: | - echo "$HOME/.local/bin" >> $GITHUB_PATH - - echo "CI_BUILD_DIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV - echo "BUILD_DIR=$GITHUB_WORKSPACE/build" >> $GITHUB_ENV - echo "DEPS_BUILD_DIR=$HOME/nvim-deps" >> $GITHUB_ENV - echo "INSTALL_PREFIX=$HOME/nvim-install" >> $GITHUB_ENV - echo "LOG_DIR=$GITHUB_WORKSPACE/build/log" >> $GITHUB_ENV - echo "NVIM_LOG_FILE=$GITHUB_WORKSPACE/build/.nvimlog" >> $GITHUB_ENV - echo "CMAKE_FLAGS=-DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX:PATH=$HOME/nvim-install -DBUSTED_OUTPUT_TYPE=nvim -DDEPS_PREFIX=$HOME/nvim-deps/usr -DMIN_LOG_LEVEL=3" >> $GITHUB_ENV - echo "ASAN_OPTIONS=detect_leaks=1:check_initialization_order=1:log_path=$GITHUB_WORKSPACE/build/log/asan" >> $GITHUB_ENV - echo "TSAN_OPTIONS=log_path=$GITHUB_WORKSPACE/build/log/tsan" >> $GITHUB_ENV - echo "UBSAN_OPTIONS=print_stacktrace=1 log_path=$GITHUB_WORKSPACE/build/log/ubsan" >> $GITHUB_ENV - echo "VALGRIND_LOG=$GITHUB_WORKSPACE/build/log/valgrind-%p.log" >> $GITHUB_ENV - echo "CACHE_NVIM_DEPS_DIR=$HOME/.cache/nvim-deps" >> $GITHUB_ENV - echo "CACHE_MARKER=$HOME/.cache/nvim-deps/.ci_cache_marker" >> $GITHUB_ENV - echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV + run: ./.github/workflows/env.sh asan - name: Setup clang repository run: | @@ -53,12 +25,6 @@ jobs: - name: Install new clang run: sudo apt-get install -y clang-11 - - name: Set ASAN env vars - run: | - echo "CLANG_SANITIZER=ASAN_UBSAN" >> $GITHUB_ENV - echo "CMAKE_FLAGS=-DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX:PATH=$HOME/nvim-install -DBUSTED_OUTPUT_TYPE=nvim -DDEPS_PREFIX=$HOME/nvim-deps/usr -DMIN_LOG_LEVEL=3 -DPREFER_LUA=ON" >> $GITHUB_ENV - echo "SYMBOLIZER=asan_symbolize-11" >> $GITHUB_ENV - - name: Setup interpreter packages run: | ./ci/before_install.sh @@ -86,29 +52,11 @@ jobs: lint: runs-on: ubuntu-latest - env: - CI_TARGET: lint steps: - uses: actions/checkout@v2 - name: Setup commom environment variables - run: | - echo "$HOME/.local/bin" >> $GITHUB_PATH - - echo "CI_BUILD_DIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV - echo "BUILD_DIR=$GITHUB_WORKSPACE/build" >> $GITHUB_ENV - echo "DEPS_BUILD_DIR=$HOME/nvim-deps" >> $GITHUB_ENV - echo "INSTALL_PREFIX=$HOME/nvim-install" >> $GITHUB_ENV - echo "LOG_DIR=$GITHUB_WORKSPACE/build/log" >> $GITHUB_ENV - echo "NVIM_LOG_FILE=$GITHUB_WORKSPACE/build/.nvimlog" >> $GITHUB_ENV - echo "CMAKE_FLAGS=-DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX:PATH=$HOME/nvim-install -DBUSTED_OUTPUT_TYPE=nvim -DDEPS_PREFIX=$HOME/nvim-deps/usr -DMIN_LOG_LEVEL=3" >> $GITHUB_ENV - echo "ASAN_OPTIONS=detect_leaks=1:check_initialization_order=1:log_path=$GITHUB_WORKSPACE/build/log/asan" >> $GITHUB_ENV - echo "TSAN_OPTIONS=log_path=$GITHUB_WORKSPACE/build/log/tsan" >> $GITHUB_ENV - echo "UBSAN_OPTIONS=print_stacktrace=1 log_path=$GITHUB_WORKSPACE/build/log/ubsan" >> $GITHUB_ENV - echo "VALGRIND_LOG=$GITHUB_WORKSPACE/build/log/valgrind-%p.log" >> $GITHUB_ENV - echo "CACHE_NVIM_DEPS_DIR=$HOME/.cache/nvim-deps" >> $GITHUB_ENV - echo "CACHE_MARKER=$HOME/.cache/nvim-deps/.ci_cache_marker" >> $GITHUB_ENV - echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV + run: ./.github/workflows/env.sh lint - name: Install apt packages run: | From 392ec5a43f85c9442097e1804f6c30d47ad0f07e Mon Sep 17 00:00:00 2001 From: James McCoy Date: Sat, 14 Nov 2020 20:36:44 -0500 Subject: [PATCH 3/5] ci: Remove $CCACHE_CPP2 The env var is being set to the default value, as of ccache 3.3, which is available in Zesty and newer. All of our CI is using Bionic, so this is already available. --- .github/workflows/env.sh | 1 - .travis.yml | 2 -- 2 files changed, 3 deletions(-) diff --git a/.github/workflows/env.sh b/.github/workflows/env.sh index 8e005c7266..84f26f949e 100755 --- a/.github/workflows/env.sh +++ b/.github/workflows/env.sh @@ -24,7 +24,6 @@ DEPS_CMAKE_FLAGS=-DUSE_BUNDLED_GPERF=OFF FUNCTIONALTEST=functionaltest CCACHE_COMPRESS=1 CCACHE_SLOPPINESS=time_macros,file_macro -CCACHE_CPP2=1 EOF BUILD_FLAGS="CMAKE_FLAGS=-DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX:PATH=$HOME/nvim-install -DBUSTED_OUTPUT_TYPE=nvim -DDEPS_PREFIX=$HOME/nvim-deps/usr -DMIN_LOG_LEVEL=3" diff --git a/.travis.yml b/.travis.yml index aad9b38934..2f4603fa5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,8 +54,6 @@ env: - CCACHE_COMPRESS=1 - CCACHE_SLOPPINESS=time_macros,file_macro - CCACHE_BASEDIR="$TRAVIS_BUILD_DIR" - # Default since 3.3, but Travis (Xenial) has 3.2.4; required with newer gcc/clang. - - CCACHE_CPP2=1 anchors: envs: &common-job-env From 3b0a729f9c39cc94084a25f2b4e740b86e4f0532 Mon Sep 17 00:00:00 2001 From: James McCoy Date: Sat, 14 Nov 2020 21:03:26 -0500 Subject: [PATCH 4/5] GHA: Use matrix to de-duplicate workflow definition The general skeleton of each job is the same, so consolidate the definition to a single job with an explicit matrix. Use conditional steps to handle steps that are unique to a certain job (e.g., installing clang-11 for better ASAN support). --- .github/workflows/linux.yml | 58 +++++++++---------------------------- 1 file changed, 14 insertions(+), 44 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f8e6556b91..4e74ac21ee 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -2,17 +2,26 @@ name: Linux CI on: [push, pull_request] jobs: - ASAN: + linux: + name: ${{ matrix.flavor }} (cc=${{ matrix.cc }}) runs-on: ubuntu-latest + strategy: + matrix: + include: + - flavor: asan + cc: clang-11 + - flavor: lint + cc: gcc env: - CC: clang-11 + CC: ${{ matrix.cc }} steps: - uses: actions/checkout@v2 - name: Setup commom environment variables - run: ./.github/workflows/env.sh asan + run: ./.github/workflows/env.sh ${{ matrix.flavor }} - name: Setup clang repository + if: matrix.flavor == 'asan' run: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - sudo add-apt-repository 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' @@ -23,6 +32,7 @@ jobs: sudo apt-get install -y autoconf automake build-essential ccache cmake cpanminus cscope gcc-multilib gdb gettext gperf language-pack-tr libtool-bin locales ninja-build pkg-config python python-pip python-setuptools python3 python3-pip python3-setuptools unzip valgrind xclip - name: Install new clang + if: matrix.flavor == 'asan' run: sudo apt-get install -y clang-11 - name: Setup interpreter packages @@ -32,51 +42,11 @@ jobs: - name: Cache dependencies uses: actions/cache@v2 - env: - cache-name: asan-deps with: path: | ${{ env.CACHE_NVIM_DEPS_DIR }} ~/.ccache - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('cmake/*', 'third-party/**', '**/CMakeLists.txt') }}-${{ github.base_ref }} - - - name: Build third-party - run: ./ci/before_script.sh - - - name: Build and test - run: ./ci/script.sh - - - name: Cache dependencies - if: ${{ success() }} - run: ./ci/before_cache.sh - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Setup commom environment variables - run: ./.github/workflows/env.sh lint - - - name: Install apt packages - run: | - sudo apt-get update - sudo apt-get install -y autoconf automake build-essential ccache cmake gcc-multilib gettext gperf libtool-bin locales ninja-build pkg-config python3 python3-pip python3-setuptools unzip - - - name: Setup interpreter packages - run: | - ./ci/before_install.sh - ./ci/install.sh - - - name: Cache dependencies - uses: actions/cache@v2 - env: - cache-name: lint-deps - with: - path: | - ${{ env.CACHE_NVIM_DEPS_DIR }} - ~/.ccache - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('cmake/*', 'third-party/**', '**/CMakeLists.txt') }}-${{ github.base_ref }} + key: ${{ runner.os }}-${{ matrix.flavor }}-${{ matrix.cc }}-${{ hashFiles('cmake/*', 'third-party/**', '**/CMakeLists.txt') }}-${{ github.base_ref }} - name: Build third-party run: ./ci/before_script.sh From af88b0891edd8d79e4586752c7007bb0e3cea28c Mon Sep 17 00:00:00 2001 From: James McCoy Date: Sat, 14 Nov 2020 21:14:30 -0500 Subject: [PATCH 5/5] GHA: Add tsan job to the matrix --- .github/workflows/linux.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4e74ac21ee..d693f5e593 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -12,6 +12,8 @@ jobs: cc: clang-11 - flavor: lint cc: gcc + - flavor: tsan + cc: clang-11 env: CC: ${{ matrix.cc }} steps: @@ -21,7 +23,7 @@ jobs: run: ./.github/workflows/env.sh ${{ matrix.flavor }} - name: Setup clang repository - if: matrix.flavor == 'asan' + if: matrix.flavor == 'asan' || matrix.flavor == 'tsan' run: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - sudo add-apt-repository 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' @@ -32,7 +34,7 @@ jobs: sudo apt-get install -y autoconf automake build-essential ccache cmake cpanminus cscope gcc-multilib gdb gettext gperf language-pack-tr libtool-bin locales ninja-build pkg-config python python-pip python-setuptools python3 python3-pip python3-setuptools unzip valgrind xclip - name: Install new clang - if: matrix.flavor == 'asan' + if: matrix.flavor == 'asan' || matrix.flavor == 'tsan' run: sudo apt-get install -y clang-11 - name: Setup interpreter packages