ci: Better core dump checking

- Do not exclude any directories from `find` search, remove dumps before tests
  instead.
- Install `apport` on travis so that linux tests should produce core dumps
  (based on information from travis-ci/travis-ci#3754, not sure whether it still
  applies).
- Check cores in lua so that one has an idea which test is failing exactly. Do
  this only 10% of time on linux because traversing the file system is slow.

Unit tests are still not touched, though it is what `app` argument in
`check_cores` is for.

TODO? consider using `find`, it may be faster. Consider retiring `os.execute`,
      dealing with escaping is bad.

.ci/common/test.sh

@@ -1,10 +1,29 @@
+print_core() {
+  local app="$1"
+  local core="$2"
+  if test "$app" = quiet ; then
+    echo "Found core $core"
+    return 0
+  fi
+  echo "======= Core file $core ======="
+  if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+    lldb -Q -o "bt all" -f "${app}" -c "${core}"
+  else
+    gdb -n -batch -ex 'thread apply all bt full' "${app}" -c "${core}"
+  fi
+}
+
 check_core_dumps() {
+  local del=
+  if test "$1" = "--delete" ; then
+    del=1
+    shift
+  fi
   local app="${1:-${BUILD_DIR}/bin/nvim}"
   if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
     local cores="$(find /cores/ -type f -print)"
   else
-    # FIXME (fwalch): Will trigger if a file named core.* exists outside of $DEPS_BUILD_DIR.
-    local cores="$(find ./ -type f -not -path "*${DEPS_BUILD_DIR}*" -name 'core.*' -print)"
+    local cores="$(find ./ -type f -name 'core.*' -print)"
   fi
 
   if [ -z "${cores}" ]; then
@@ -12,12 +31,16 @@ check_core_dumps() {
   fi
   local core
   for core in $cores; do
-    if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
-      lldb -Q -o "bt all" -f "${app}" -c "${core}"
+    if test "$del" = "1" ; then
+      print_core "$app" "$core" >&2
+      rm "$core"
     else
-      gdb -n -batch -ex 'thread apply all bt full' "${app}" -c "${core}"
+      print_core "$app" "$core"
     fi
   done
+  if test "$app" = quiet ; then
+    return 0
+  fi
   exit 1
 }
 

.ci/run_tests.sh

@@ -7,6 +7,8 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/test.sh"
 
+check_core_dumps --delete quiet
+
 prepare_build
 build_nvim
 

.travis.yml

@@ -98,6 +98,7 @@ addons:
     packages:
       - autoconf
       - automake
+      - apport
       - build-essential
       - clang-3.8
       - cmake

test/functional/helpers.lua

@@ -9,6 +9,7 @@ local TcpStream = require('nvim.tcp_stream')
 local SocketStream = require('nvim.socket_stream')
 local ChildProcessStream = require('nvim.child_process_stream')
 
+local check_cores = global_helpers.check_cores
 local check_logs = global_helpers.check_logs
 local neq = global_helpers.neq
 local eq = global_helpers.eq
@@ -608,7 +609,10 @@ local M = {
 
 return function(after_each)
   if after_each then
-    after_each(check_logs)
+    after_each(function()
+      check_logs()
+      check_cores('build/bin/nvim')
+    end)
   end
   return M
 end

test/helpers.lua

@@ -17,6 +17,34 @@ local ok = function(res)
   return assert.is_true(res)
 end
 
+local function glob(initial_path, re, exc_re)
+  local paths_to_check = {initial_path}
+  local ret = {}
+  local checked_files = {}
+  while #paths_to_check > 0 do
+    local cur_path = paths_to_check[#paths_to_check]
+    paths_to_check[#paths_to_check] = nil
+    for e in lfs.dir(cur_path) do
+      local full_path = cur_path .. '/' .. e
+      local checked_path = full_path:sub(#initial_path + 1)
+      if ((not exc_re or not checked_path:match(exc_re))
+          and e:sub(1, 1) ~= '.') then
+        local attrs = lfs.attributes(full_path)
+        local check_key = attrs.dev .. ':' .. tostring(attrs.ino)
+        if not checked_files[check_key] then
+          checked_files[check_key] = true
+          if attrs.mode == 'directory' then
+            paths_to_check[#paths_to_check + 1] = full_path
+          elseif not re or checked_path:match(re) then
+            ret[#ret + 1] = full_path
+          end
+        end
+      end
+    end
+  end
+  return ret
+end
+
 local function check_logs()
   local log_dir = os.getenv('LOG_DIR')
   local runtime_errors = 0
@@ -109,6 +137,79 @@ local function filter(filter_func, tab)
   return rettab
 end
 
+local function hasenv(name)
+  local env = os.getenv(name)
+  if env and env ~= '' then
+    return env
+  end
+  return nil
+end
+
+local tests_skipped = 0
+
+local function check_cores(app)
+  app = app or 'build/bin/nvim'
+  local initial_path, re, exc_re
+  local gdb_db_cmd = 'gdb -n -batch -ex "thread apply all bt full" "$_NVIM_TEST_APP" -c "$_NVIM_TEST_CORE"'
+  local lldb_db_cmd = 'lldb -Q -o "bt all" -f "$_NVIM_TEST_APP" -c "$_NVIM_TEST_CORE"'
+  local random_skip = false
+  local db_cmd
+  if hasenv('NVIM_TEST_CORE_GLOB_DIRECTORY') then
+    initial_path = os.getenv('NVIM_TEST_CORE_GLOB_DIRECTORY')
+    re = os.getenv('NVIM_TEST_CORE_GLOB_RE')
+    exc_re = os.getenv('NVIM_TEST_CORE_EXC_RE')
+    db_cmd = os.getenv('NVIM_TEST_CORE_DB_CMD') or gdb_db_cmd
+    random_skip = os.getenv('NVIM_TEST_CORE_RANDOM_SKIP')
+  elseif os.getenv('TRAVIS_OS_NAME') == 'osx' then
+    initial_path = '/cores'
+    re = nil
+    exc_re = nil
+    db_cmd = lldb_db_cmd
+  else
+    initial_path = '.'
+    re = 'core[^/]*'
+    exc_re = '^/%.deps$'
+    db_cmd = gdb_db_cmd
+    random_skip = true
+  end
+  -- Finding cores takes too much time on linux
+  if random_skip and math.random() < 0.9 then
+    tests_skipped = tests_skipped + 1
+    return
+  end
+  local cores = glob(initial_path, re, exc_re)
+  local found_cores = 0
+  local out = io.stdout
+  for _, core in ipairs(cores) do
+    local len = 80 - #core - #('Core file ') - 2
+    local esigns = ('='):rep(len / 2)
+    out:write(('\n%s Core file %s %s\n'):format(esigns, core, esigns))
+    out:flush()
+    local pipe = io.popen(
+        db_cmd:gsub('%$_NVIM_TEST_APP', app):gsub('%$_NVIM_TEST_CORE', core)
+        .. ' 2>&1', 'r')
+    if pipe then
+      local bt = pipe:read('*a')
+      if bt then
+        out:write(bt)
+        out:write('\n')
+      else
+        out:write('Failed to read from the pipe\n')
+      end
+    else
+      out:write('Failed to create pipe\n')
+    end
+    out:flush()
+    found_cores = found_cores + 1
+    os.remove(core)
+  end
+  if found_cores ~= 0 then
+    out:write(('\nTests covered by this check: %u\n'):format(tests_skipped + 1))
+  end
+  tests_skipped = 0
+  assert(0 == found_cores)
+end
+
 return {
   eq = eq,
   neq = neq,
@@ -118,4 +219,7 @@ return {
   tmpname = tmpname,
   map = map,
   filter = filter,
+  glob = glob,
+  check_cores = check_cores,
+  hasenv = hasenv,
 }