FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

tools: add more options to track flaky tests · nodejs/node@a1d5209 · GitHub

/ node Public

Commit a1d5209

Browse files
aduh95
authored and
danielleadams
committed
tools: add more options to track flaky tests
Refs: #43929 (comment) PR-URL: #43954 Backport-PR-URL: #45126 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Tobias Nießen <tniessen@tnie.de> Reviewed-By: Feng Yu <F3n67u@outlook.com>
1 parent 98c49d8 commit a1d5209

7 files changed

Lines changed: 40 additions & 24 deletions

File tree

‎.github/workflows/build-tarball.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,11 @@ on:
1111
- v[0-9]+.x
1212

1313
env:
14-
FLAKY_TESTS: dontcare
14+
PYTHON_VERSION: '3.10'
15+
FLAKY_TESTS: keep_retrying
1516

1617
jobs:
1718
build-tarball:
18-
env:
19-
PYTHON_VERSION: '3.10'
2019
runs-on: ubuntu-latest
2120
steps:
2221
- uses: actions/checkout@v2
@@ -71,4 +70,4 @@ jobs:
7170
- name: Test
7271
run: |
7372
cd $TAR_DIR
74-
make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
73+
make run-ci -j2 V=1 TEST_CI_ARGS="-p dots --measure-flakiness 9"

‎.github/workflows/build-windows.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313

1414
env:
1515
PYTHON_VERSION: '3.10'
16-
FLAKY_TESTS: dontcare
16+
FLAKY_TESTS: keep_retrying
1717

1818
jobs:
1919
build-windows:

‎.github/workflows/misc.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,4 @@ jobs:
3131
name: docs
3232
path: out/doc
3333
- name: Test
34-
run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions"
34+
run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions --measure-flakiness 9"

‎.github/workflows/test-asan.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919

2020
env:
2121
PYTHON_VERSION: '3.10'
22-
FLAKY_TESTS: dontcare
22+
FLAKY_TESTS: keep_retrying
2323

2424
jobs:
2525
test-asan:
@@ -40,4 +40,4 @@ jobs:
4040
- name: Build
4141
run: make build-ci -j2 V=1
4242
- name: Test
43-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300"
43+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300 --measure-flakiness 9"

‎.github/workflows/test-linux.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313

1414
env:
1515
PYTHON_VERSION: '3.10'
16-
FLAKY_TESTS: dontcare
16+
FLAKY_TESTS: keep_retrying
1717

1818
jobs:
1919
test-linux:
@@ -29,4 +29,4 @@ jobs:
2929
- name: Build
3030
run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
3131
- name: Test
32-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
32+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9"

‎.github/workflows/test-macos.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919

2020
env:
2121
PYTHON_VERSION: '3.10'
22-
FLAKY_TESTS: dontcare
22+
FLAKY_TESTS: keep_retrying
2323

2424
jobs:
2525
test-macOS:
@@ -35,4 +35,4 @@ jobs:
3535
- name: Build
3636
run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
3737
- name: Test
38-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
38+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9"

‎tools/test.py‎

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,11 @@ def get_module(name, path):
9696

9797
class ProgressIndicator(object):
9898

99-
def __init__(self, cases, flaky_tests_mode):
99+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
100100
self.cases = cases
101101
self.serial_id = 0
102102
self.flaky_tests_mode = flaky_tests_mode
103+
self.measure_flakiness = measure_flakiness
103104
self.parallel_queue = Queue(len(cases))
104105
self.sequential_queue = Queue(len(cases))
105106
for case in cases:
@@ -211,10 +212,22 @@ def RunSingle(self, parallel, thread_id):
211212
if output.UnexpectedOutput():
212213
if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE:
213214
self.flaky_failed.append(output)
215+
elif FLAKY in output.test.outcomes and self.flaky_tests_mode == KEEP_RETRYING:
216+
for _ in range(99):
217+
if not case.Run().UnexpectedOutput():
218+
self.flaky_failed.append(output)
219+
break
220+
else:
221+
# If after 100 tries, the test is not passing, it's not flaky.
222+
self.failed.append(output)
214223
else:
215224
self.failed.append(output)
216225
if output.HasCrashed():
217226
self.crashed += 1
227+
if self.measure_flakiness:
228+
outputs = [case.Run() for _ in range(self.measure_flakiness)]
229+
# +1s are there because the test already failed once at this point.
230+
print(" failed {} out of {}".format(len([i for i in outputs if i.UnexpectedOutput()]) + 1, self.measure_flakiness + 1))
218231
else:
219232
self.succeeded += 1
220233
self.remaining -= 1
@@ -436,8 +449,8 @@ def Done(self):
436449

437450
class CompactProgressIndicator(ProgressIndicator):
438451

439-
def __init__(self, cases, flaky_tests_mode, templates):
440-
super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode)
452+
def __init__(self, cases, flaky_tests_mode, measure_flakiness, templates):
453+
super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness)
441454
self.templates = templates
442455
self.last_status_length = 0
443456
self.start_time = time.time()
@@ -492,29 +505,29 @@ def PrintProgress(self, name):
492505

493506
class ColorProgressIndicator(CompactProgressIndicator):
494507

495-
def __init__(self, cases, flaky_tests_mode):
508+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
496509
templates = {
497510
'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
498511
'stdout': "\033[1m%s\033[0m",
499512
'stderr': "\033[31m%s\033[0m",
500513
}
501-
super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
514+
super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates)
502515

503516
def ClearLine(self, last_line_length):
504517
print("\033[1K\r", end='')
505518

506519

507520
class MonochromeProgressIndicator(CompactProgressIndicator):
508521

509-
def __init__(self, cases, flaky_tests_mode):
522+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
510523
templates = {
511524
'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
512525
'stdout': '%s',
513526
'stderr': '%s',
514527
'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
515528
'max_length': 78
516529
}
517-
super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
530+
super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates)
518531

519532
def ClearLine(self, last_line_length):
520533
print(("\r" + (" " * last_line_length) + "\r"), end='')
@@ -946,8 +959,8 @@ def GetVm(self, arch, mode):
946959
def GetTimeout(self, mode):
947960
return self.timeout * TIMEOUT_SCALEFACTOR[ARCH_GUESS or 'ia32'][mode]
948961

949-
def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
950-
progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode)
962+
def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode, measure_flakiness):
963+
progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode, measure_flakiness)
951964
return progress.Run(tasks)
952965

953966
# -------------------------------------------
@@ -965,6 +978,7 @@ def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
965978
SLOW = 'slow'
966979
FLAKY = 'flaky'
967980
DONTCARE = 'dontcare'
981+
KEEP_RETRYING = 'keep_retrying'
968982

969983
class Expression(object):
970984
pass
@@ -1353,8 +1367,11 @@ def BuildOptions():
13531367
result.add_option("--cat", help="Print the source of the tests",
13541368
default=False, action="store_true")
13551369
result.add_option("--flaky-tests",
1356-
help="Regard tests marked as flaky (run|skip|dontcare)",
1370+
help="Regard tests marked as flaky (run|skip|dontcare|keep_retrying)",
13571371
default="run")
1372+
result.add_option("--measure-flakiness",
1373+
help="When a test fails, re-run it x number of times",
1374+
default=0, type="int")
13581375
result.add_option("--skip-tests",
13591376
help="Tests that should not be executed (comma-separated)",
13601377
default="")
@@ -1426,7 +1443,7 @@ def ProcessOptions(options):
14261443
# tends to exaggerate the number of available cpus/cores.
14271444
cores = os.environ.get('JOBS')
14281445
options.j = int(cores) if cores is not None else multiprocessing.cpu_count()
1429-
if options.flaky_tests not in [RUN, SKIP, DONTCARE]:
1446+
if options.flaky_tests not in [RUN, SKIP, DONTCARE, KEEP_RETRYING]:
14301447
print("Unknown flaky-tests mode %s" % options.flaky_tests)
14311448
return False
14321449
return True
@@ -1726,7 +1743,7 @@ def should_keep(case):
17261743
else:
17271744
try:
17281745
start = time.time()
1729-
if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests):
1746+
if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests, options.measure_flakiness):
17301747
result = 0
17311748
else:
17321749
result = 1

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL