patch 9.1.0820: tests: Mac OS tests are too flaky
Commit:
https://github.com/vim/vim/commit/baab7c08653a4e1b7227d6426d96cab030ccf9e8
Author: Milly <milly...@gmail.com>
Date: Mon Oct 28 21:56:14 2024 +0100
patch 9.1.0820: tests: Mac OS tests are too flaky
Problem: tests: Mac OS tests are too flaky
Solution: Increase max test timeout to 25 minutes,
allow up to 10 retries on Mac OS runners,
refactor runtest.vim (Milly).
closes: #15940
Co-authored-by: K.Takata <ken...@csc.jp>
Signed-off-by: Milly <milly...@gmail.com>
Signed-off-by: Christian Brabandt <c...@256bit.org>
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c1cca4056..6b3b3d0fb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -389,7 +389,7 @@ jobs:
brew install diffutils
- name: Test
- timeout-minutes: 20
+ timeout-minutes: 25
run: |
make ${TEST}
diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim
index 7ac48455e..ded31975b 100644
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -190,10 +190,6 @@ function GetAllocId(name)
return lnum - top - 1
endfunc
-if has('reltime')
- let g:func_start = reltime()
-endif
-
" Get the list of swap files in the current directory.
func s:GetSwapFileList()
let save_dir = &directory
@@ -603,6 +599,16 @@ for g:testfunc in sort(s:tests)
" A test can set g:test_is_flaky to retry running the test.
let g:test_is_flaky = 0
+ " A test can set g:max_run_nr to change the max retry count.
+ let g:max_run_nr = 5
+ if has('mac')
+ let g:max_run_nr = 10
+ endif
+
+ " By default, give up if the same error occurs. A test can set
+ " g:giveup_same_error to 0 to not give up on the same error and keep trying.
+ let g:giveup_same_error = 1
+
let starttime = strftime("%H:%M:%S")
call RunTheTest(g:testfunc)
@@ -618,10 +624,15 @@ for g:testfunc in sort(s:tests)
call extend(s:messages, v:errors)
let endtime = strftime("%H:%M:%S")
- call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}:')
+ if has('reltime')
+ let suffix = $' in{reltimestr(reltime(g:func_start))} seconds'
+ else
+ let suffix = ''
+ endif
+ call add(total_errors, $'Run {g:run_nr}, {starttime} -
{endtime}{suffix}:')
call extend(total_errors, v:errors)
- if g:run_nr >= 5 || prev_error == v:errors[0]
+ if g:run_nr >= g:max_run_nr || g:giveup_same_error && prev_error ==
v:errors[0]
call add(total_errors, 'Flaky test failed too often, giving up')
let v:errors = total_errors
break
@@ -632,7 +643,8 @@ for g:testfunc in sort(s:tests)
" Flakiness is often caused by the system being very busy. Sleep a
" couple of seconds to have a higher chance of succeeding the second
" time.
- sleep 2
+ let delay = g:run_nr * 2
+ exe 'sleep' delay
let prev_error = v:errors[0]
let v:errors = []
diff --git a/src/testdir/screendump.vim b/src/testdir/screendump.vim
index e7cda2ef4..84b486441 100644
--- a/src/testdir/screendump.vim
+++ b/src/testdir/screendump.vim
@@ -56,6 +56,7 @@ func VerifyScreenDump(buf, filename, options, ...)
" Starting a terminal to make a screendump is always considered flaky.
let g:test_is_flaky = 1
+ let g:giveup_same_error = 0
" wait for the pending updates to be handled.
call TermWait(a:buf)
@@ -83,41 +84,55 @@ func VerifyScreenDump(buf, filename, options, ...)
sleep 50m
call delete(testfile)
call term_dumpwrite(a:buf, testfile, a:options)
+
+ if refdump->empty()
+ let msg = 'See new dump file: call term_dumpload("testdir/' .. testfile
.. '")'
+ call assert_report(msg)
+ " no point in retrying
+ let g:run_nr = 10
+ return 1
+ endif
+
let testdump = ReadAndFilter(testfile, filter)
if refdump == testdump
call delete(testfile)
if did_mkdir
call delete('failed', 'd')
endif
+ if i > 0
+ call remove(v:errors, -1)
+ endif
break
endif
- if i == max_loops
- " Leave the failed dump around for inspection.
- if filereadable(reference)
- let msg = 'See dump file difference: call term_dumpdiff("testdir/' ..
testfile .. '", "testdir/' .. reference .. '")'
- if a:0 == 1
- let msg = a:1 . ': ' . msg
- endif
- if len(testdump) != len(refdump)
- let msg = msg . '; line count is ' . len(testdump) . ' instead of ' .
len(refdump)
- endif
- else
- let msg = 'See new dump file: call term_dumpload("testdir/' .. testfile
.. '")'
- " no point in retrying
- let g:run_nr = 10
+
+ " Leave the failed dump around for inspection.
+ let msg = 'See dump file difference: call term_dumpdiff("testdir/' ..
testfile .. '", "testdir/' .. reference .. '")'
+ if a:0 == 1
+ let msg = a:1 . ': ' . msg
+ endif
+ if len(testdump) != len(refdump)
+ let msg = msg . '; line count is ' . len(testdump) . ' instead of ' .
len(refdump)
+ endif
+ for j in range(len(refdump))
+ if j >= len(testdump)
+ break
endif
- for i in range(len(refdump))
- if i >= len(testdump)
- break
- endif
- if testdump[i] != refdump[i]
- let msg = msg . '; difference in line ' . (i + 1) . ': "' .
testdump[i] . '"'
- endif
- endfor
- call assert_report(msg)
- return 1
+ if testdump[j] != refdump[j]
+ let msg = msg . '; difference in line ' . (j + 1) . ': "' . testdump[j]
. '"'
+ endif
+ endfor
+
+ " Always add the last error so that it is displayed on timeout.
+ " See TestTimeout() in runtest.vim.
+ if i > 0
+ call remove(v:errors, -1)
endif
+ call assert_report(msg)
+
let i += 1
+ if i >= max_loops
+ return 1
+ endif
endwhile
return 0
endfunc
diff --git a/src/testdir/test_channel.vim b/src/testdir/test_channel.vim
index 78c834d70..2d40eff12 100644
--- a/src/testdir/test_channel.vim
+++ b/src/testdir/test_channel.vim
@@ -2762,6 +2762,8 @@ func LspTests(port)
endfunc
func Test_channel_lsp_mode()
+ " The channel lsp mode test is flaky and gives the same error.
+ let g:giveup_same_error = 0
call RunServer('test_channel_lsp.py', 'LspTests', [])
endfunc
diff --git a/src/version.c b/src/version.c
index 838f25acd..e04296567 100644
--- a/src/version.c
+++ b/src/version.c
@@ -704,6 +704,8 @@ static char *(features[]) =
static int included_patches[] =
{ /* Add new patch number below this line */
+/**/
+ 820,
/**/
819,
/**/
--
--
You received this message from the "vim_dev" maillist.
Do not top-post! Type your reply below the text you are replying to.
For more information, visit http://www.vim.org/maillist.php
---
You received this message because you are subscribed to the Google Groups
"vim_dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to vim_dev+unsubscr...@googlegroups.com.
To view this discussion visit
https://groups.google.com/d/msgid/vim_dev/E1t5WqK-00EMfP-Ok%40256bit.org.
