oldtests: Allow running old tests in parallel

This actually makes sense only on developer machines and adds overhead (copying all test files for each test run), but not much of that.
neovim · ZyX-I · Jan 23, 2018 · Jan 26, 2018 · Jan 27, 2018 · Feb 1, 2018
commit f449ec88a62249406c89d8e3b5b2197bcdfbaf24
diff --git a/Makefile b/Makefile
@@ -55,10 +55,6 @@ ifneq (,$(findstring functionaltest-lua,$(MAKECMDGOALS)))
   $(shell [ -x $(DEPS_BUILD_DIR)/usr/bin/lua ] || rm build/.ran-*)
 endif
 
-# For use where we want to make sure only a single job is run.  This does issue 
-# a warning, but we need to keep SCRIPTS argument.
-SINGLE_MAKE = export MAKEFLAGS= ; $(MAKE)
-
 all: nvim
 
 nvim: build/.ran-cmake deps
@@ -92,11 +88,11 @@ endif
 
 # TODO: cmake 3.2+ add_custom_target() has a USES_TERMINAL flag.
 oldtest: | nvim helptags
-	+$(SINGLE_MAKE) -C src/nvim/testdir clean
+	+$(MAKE) -C src/nvim/testdir clean
 ifeq ($(strip $(TEST_FILE)),)
-	+$(SINGLE_MAKE) -C src/nvim/testdir NVIM_PRG="$(realpath build/bin/nvim)" $(MAKEOVERRIDES)
+	+$(MAKE) -C src/nvim/testdir NVIM_PRG="$(realpath build/bin/nvim)" $(MAKEOVERRIDES)
 else
-	+$(SINGLE_MAKE) -C src/nvim/testdir NVIM_PRG="$(realpath build/bin/nvim)" NEW_TESTS=$(TEST_FILE) SCRIPTS= $(MAKEOVERRIDES)
+	+$(MAKE) -C src/nvim/testdir NVIM_PRG="$(realpath build/bin/nvim)" NEW_TESTS=$(TEST_FILE) SCRIPTS= $(MAKEOVERRIDES)
 endif
 
 helptags: | nvim

diff --git a/src/nvim/testdir/Makefile b/src/nvim/testdir/Makefile
@@ -7,11 +7,13 @@ ifeq ($(OS),Windows_NT)
 else
   NVIM_PRG ?= ../../../build/bin/nvim
 endif
-ROOT := ../../..
+ROOT := $(abspath ../../..)
+BUILD_DIR := $(abspath $(dir $(NVIM_PRG))/..)
+NVIM_PRG_ABS := $(abspath $(NVIM_PRG))
+CTDIR := $(BUILD_DIR)/oldtests
+CTLOG := $(CTDIR)/test.log
 
 export SHELL := sh
-export NVIM_PRG := $(NVIM_PRG)
-export TMPDIR := $(abspath ../../../Xtest-tmpdir)
 
 SCRIPTS_DEFAULT = \
            test14.out             \
@@ -160,78 +162,47 @@ gui: $(SCRIPTS) $(SCRIPTS_GUI) newtests
 report:
 	@echo
 	@echo 'Test results:'
-	@/bin/sh -c "if test -f test.log; then \
-	                 cat test.log;         \
-	                 echo TEST FAILURE;    \
-	                 exit 1;               \
-	             else                      \
-	                 echo ALL DONE;        \
-	             fi"
+	@/bin/sh runnvim.sh --report $(ROOT) $(BUILD_DIR)
 
 test1.out: nolog $(NVIM_PRG)
 
 $(SCRIPTS) $(SCRIPTS_GUI): test1.out
 
 $(NEW_TESTS): test1.out
 
-RM_ON_RUN   := test.out X* viminfo
 RM_ON_START := test.ok
-RUN_VIM     := $(TOOL) $(NVIM_PRG) -u unix.vim -U NONE -i viminfo --headless --noplugin -s dotest.in
+RUN_VIM     := $(TOOL) $(NVIM_PRG_ABS) -u unix.vim -U NONE -i viminfo --headless --noplugin -s dotest.in
 
 clean:
-	-rm -rf *.out          \
-	        *.failed       \
-	        *.res          \
-	        *.rej          \
-	        *.orig         \
-	        *.tlog         \
-	        test.log       \
-	        messages       \
-	        $(RM_ON_RUN)   \
-	        $(RM_ON_START) \
-	        valgrind.*     \
-	        .*.swp         \
-	        .*.swo         \
-	        .gdbinit       \
-	        $(TMPDIR)      \
-	        del
+	-rm -rf $(CTDIR)
 
 test1.out: .gdbinit test1.in
 	@echo "[OLDTEST-PREP] Running test1"
-	@rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
-	@mkdir -p $(TMPDIR)
-	@/bin/sh runnvim.sh $(ROOT) $(NVIM_PRG) $* $(RUN_VIM) $*.in
-	@rm -f wrongtermsize
-	@rm -rf X* viminfo
+	@/bin/sh runnvim.sh $(ROOT) $(BUILD_DIR) $(NVIM_PRG_ABS) $* \
+	  $(RUN_VIM) $*.in
+	@! test -e wrongtermsize
 
 %.out: %.in .gdbinit
 	@echo "[OLDESTTEST] Running" $*
-	@rm -rf $*.failed test.ok $(RM_ON_RUN)
-	@mkdir -p $(TMPDIR)
-	@cp $*.ok test.ok
-	@/bin/sh runnvim.sh --oldesttest $(ROOT) $(NVIM_PRG) $* $(RUN_VIM) $*.in
-	@rm -rf X* test.ok viminfo
+	@/bin/sh runnvim.sh \
+	  --oldesttest $(ROOT) $(BUILD_DIR) $(NVIM_PRG_ABS) $* \
+	  $(RUN_VIM) $*.in
 
 test49.out: test49.vim
 
 nolog:
-	@echo "[OLDTEST-PREP] Removing test.log and messages"
-	@rm -f test.log messages
+	@echo "[OLDTEST-PREP] Removing test.log"
+	@rm -f $(CTLOG)
 
 
 # New style of tests uses Vim script with assert calls.  These are easier
 # to write and a lot easier to read and debug.
 # Limitation: Only works with the +eval feature.
-RUN_VIMTEST = $(TOOL) $(NVIM_PRG) -u unix.vim -U NONE --headless --noplugin
+RUN_VIMTEST = $(TOOL) $(NVIM_PRG_ABS) -u unix.vim -U NONE --headless --noplugin
 
-newtests: newtestssilent
-	@/bin/sh -c "if test -f messages && grep -q 'FAILED' messages; then \
-	                 cat messages && cat test.log;                      \
-	             fi"
-
-newtestssilent: $(NEW_TESTS)
+newtests: $(NEW_TESTS)
 
 %.res: %.vim .gdbinit
 	@echo "[OLDTEST] Running" $*
-	@mkdir -p $(TMPDIR)
-	@/bin/sh runnvim.sh $(ROOT) $(NVIM_PRG) $* $(RUN_VIMTEST) -u NONE -S runtest.vim $*.vim
+	@/bin/sh runnvim.sh $(ROOT) $(BUILD_DIR) $(NVIM_PRG_ABS) $* \
+	  $(RUN_VIMTEST) -u NONE -S runtest.vim $*.vim
diff --git a/src/nvim/testdir/runnvim.sh b/src/nvim/testdir/runnvim.sh
@@ -1,18 +1,89 @@
 #!/bin/sh
 
+# set -x
+set -e
+
+prepare_tdir() {
+  local tdir="$1" ; shift
+  local test_name="$1" ; shift
+
+  if test -d "$tdir" ; then
+    rm -r "$tdir"
+  fi
+  mkdir -p "$tdir"
+
+  mkdir -p "$tdir/Xtest-tmpdir"
+  export TMPDIR="$tdir/Xtest-tmpdir"
+
+  cp -a * "$tdir"
+  if test -e "$tdir/$test_name.ok" ; then
+    cp "$tdir/$test_name.ok" "$tdir/test.ok"
+  fi
+}
+
+help() {
+  echo "Usage:"
+  echo
+  echo "  $0 --help"
+  echo "  $0 --report {root} {build_dir}"
+  echo "  $0 [--oldesttest] {root} {build_dir} {nvim_prg} {test_name} {cmd}..."
+  echo
+  echo "  --help: show this help."
+  echo
+  echo "  --report: check whether test failure report file exists. If it does"
+  echo "            then output it and exit with non-zero exit code."
+  echo
+  echo "  --oldesttest: use oldest test report files to check for error."
+  echo "                Specifically will check that there is test.out and that"
+  echo "                it does not differ from {test_name}.ok. Otherwise"
+  echo "                script checks for failures in messages file."
+  echo
+  echo "  {root}: git repository root directory."
+  echo "  {build_dir}: CMake build directory, normally {root}/build."
+  echo "  {nvim_prg}: Neovim executable name, normally {build_dir}/bin/nvim."
+  echo "  {test_name}: test name without extension."
+  echo "  {cmd}: command to run to perform test and corresponding arguments."
+}
+
+report() {
+  local root="$1" ; shift
+  local build_dir="$1" ; shift
+
+  local ctdir="$build_dir/oldtests"
+  local ctlog="$ctdir/test.log"
+
+  echo "Test results"
+  if test -f "$ctlog" ; then
+    cat "$ctlog"
+    echo "TEST FAILURE"
+    return 1
+  else
+    echo "ALL DONE"
+    return 0
+  fi
+}
+
 main() {(
-  local separator="================================================================================"
+  if test "$1" = "--help" ; then
+    help
+    return 0
+  fi
+  local report=
+  if test "$1" = "--report" ; then
+    shift
+    report "$@"
+    return $?
+  fi
   local oldesttest=
   if test "$1" = "--oldesttest" ; then
     shift
     oldesttest=1
   fi
   local root="$1" ; shift
+  local build_dir="$1" ; shift
   local nvim_prg="$1" ; shift
   local test_name="$1" ; shift
 
-  local tlog="$test_name.tlog"
-
   export NVIM_TEST_ARGC=$#
   local arg
   local i=0
@@ -22,42 +93,65 @@ main() {(
   done
 
   export CI_DIR="$root/ci"
-  export BUILD_DIR="$(dirname "$nvim_prg")/.."
+  export ROOT="$root"
+  export BUILD_DIR="$build_dir"
   export FAILED=0
+  export NVIM_PRG="$nvim_prg"
 
   . "$CI_DIR/common/suite.sh"
   . "$CI_DIR/common/test.sh"
 
+  local ctdir="$build_dir/oldtests"
+  local tdir="$ctdir/$test_name"
+
+  prepare_tdir "$tdir" "$test_name"
+
+  local tlog="$ctdir/$test_name.tlog"
+  local ctlog="$ctdir/test.log"
+
   export VIMRUNTIME="$root/runtime"
+  cd "$tdir"
   if ! "$nvim_prg" \
     -u NONE -i NONE \
     --headless \
     --cmd 'set shortmess+=I noswapfile noundofile nomore' \
     -S runnvim.vim \
-    "$tlog" > "out-$tlog" 2> "err-$tlog"
+    "$tlog" > "$tlog.out" 2> "$tlog.err"
   then
     fail "$test_name" F "Nvim exited with non-zero code"
   fi
+  local separator="================================================================================"
   echo "Stdout of :terminal runner" >> "$tlog"
   echo "$separator" >> "$tlog"
-  cat "out-$tlog" >> "$tlog"
+  cat "$tlog.out" >> "$tlog"
   echo "$separator" >> "$tlog"
   echo "Stderr of :terminal runner" >> "$tlog"
   echo "$separator" >> "$tlog"
-  cat "err-$tlog" >> "$tlog"
+  cat "$tlog.err" >> "$tlog"
   echo "$separator" >> "$tlog"
   if test "$oldesttest" = 1 ; then
-    if ! diff -q test.out "$test_name.ok" > /dev/null 2>&1 ; then
-      if test -f test.out ; then
+    if ! diff -q "$tdir/test.out" "$tdir/$test_name.ok" > /dev/null 2>&1 ; then
+      if test -f "$tdir/test.out" ; then
         fail "$test_name" F "Oldest test .out file differs from .ok file"
         echo "Diff between test.out and $test_name.ok" >> "$tlog"
         echo "$separator" >> "$tlog"
-        diff -a test.out "$test_name.ok" >> "$tlog"
+        diff -a "$tdir/test.out" "$tdir/$test_name.ok" >> "$tlog" || true
         echo "$separator" >> "$tlog"
       else
         echo "No output in test.out" >> "$tlog"
       fi
     fi
+  else
+    local messages="$tdir/messages"
+    if test -e "$tdir/messages" ; then
+      if grep -q "FAILED" "$tdir/messages" ; then
+        fail "$test_name" F "Messages file contains FAILED message"
+      fi
+      echo "Messages file:" >> "$tlog"
+      echo "$separator" >> "$tlog"
+      cat "$tdir/messages" >> "$tlog"
+      echo "$separator" >> "$tlog"
+    fi
   fi
   if test "$FAILED" = 1 ; then
     travis_fold start "$NVIM_TEST_CURRENT_SUITE/$test_name"
@@ -75,7 +169,7 @@ main() {(
     travis_fold end "$NVIM_TEST_CURRENT_SUITE/$test_name"
   fi
   if test "$FAILED" = 1 ; then
-    echo "Test $test_name failed, see output above and summary for more details" >> test.log
+    echo "Test $test_name failed, see output above and summary for more details" >> "$ctlog"
   fi
 )}
 

diff --git a/src/nvim/testdir/test_findfile.vim b/src/nvim/testdir/test_findfile.vim
@@ -3,23 +3,14 @@
 func Test_findfile()
   new
   let cwd=getcwd()
-  cd ..
-
-  " Tests may be run from a shadow directory, so an extra cd needs to be done to
-  " get above src/
-  if fnamemodify(getcwd(), ':t') != 'src'
-    cd ../.. 
-  else 
-    cd .. 
-  endif
-  set ssl
+  cd $ROOT
+  set shellslash
 
   call assert_equal('src/nvim/testdir/test_findfile.vim', findfile('test_findfile.vim','src/nvim/test*'))
-  exe "cd" cwd
-  cd ..
+  cd $ROOT/src/nvim
   call assert_equal('testdir/test_findfile.vim', findfile('test_findfile.vim','test*'))
   call assert_equal('testdir/test_findfile.vim', findfile('test_findfile.vim','testdir'))
 
-  exe "cd" cwd
+  execute 'cd' fnameescape(cwd)
   q!
 endfunc