FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-98903: Test suite fails with exit code 4 if no tests ran (#98904) · python/cpython@c76db37 · GitHub

  1. python
  2. cpython

Repository navigation

Commit c76db37

Browse files
vstinner
authored
gh-98903: Test suite fails with exit code 4 if no tests ran (#98904)
The Python test suite now fails wit exit code 4 if no tests ran. It should help detecting typos in test names and test methods. * Add "EXITCODE_" constants to Lib/test/libregrtest/main.py. * Fix a typo: "NO TEST RUN" becomes "NO TESTS RAN"
1 parent 0689b99 commit c76db37

3 files changed

Lines changed: 52 additions & 26 deletions

File tree

‎Lib/test/libregrtest/main.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@
2828
# Must be smaller than buildbot "1200 seconds without output" limit.
2929
EXIT_TIMEOUT = 120.0
3030

31+
EXITCODE_BAD_TEST = 2
32+
EXITCODE_INTERRUPTED = 130
33+
EXITCODE_ENV_CHANGED = 3
34+
EXITCODE_NO_TESTS_RAN = 4
35+
3136

3237
class Regrtest:
3338
"""Execute a test suite.
@@ -493,15 +498,18 @@ def display_header(self):
493498
print("== encodings: locale=%s, FS=%s"
494499
% (locale.getencoding(), sys.getfilesystemencoding()))
495500

501+
def no_tests_run(self):
502+
return not any((self.good, self.bad, self.skipped, self.interrupted,
503+
self.environment_changed))
504+
496505
def get_tests_result(self):
497506
result = []
498507
if self.bad:
499508
result.append("FAILURE")
500509
elif self.ns.fail_env_changed and self.environment_changed:
501510
result.append("ENV CHANGED")
502-
elif not any((self.good, self.bad, self.skipped, self.interrupted,
503-
self.environment_changed)):
504-
result.append("NO TEST RUN")
511+
elif self.no_tests_run():
512+
result.append("NO TESTS RAN")
505513

506514
if self.interrupted:
507515
result.append("INTERRUPTED")
@@ -750,11 +758,13 @@ def _main(self, tests, kwargs):
750758
self.save_xml_result()
751759

752760
if self.bad:
753-
sys.exit(2)
761+
sys.exit(EXITCODE_BAD_TEST)
754762
if self.interrupted:
755-
sys.exit(130)
763+
sys.exit(EXITCODE_INTERRUPTED)
756764
if self.ns.fail_env_changed and self.environment_changed:
757-
sys.exit(3)
765+
sys.exit(EXITCODE_ENV_CHANGED)
766+
if self.no_tests_run():
767+
sys.exit(EXITCODE_NO_TESTS_RAN)
758768
sys.exit(0)
759769

760770

‎Lib/test/test_regrtest.py‎

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@
2828
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
2929
LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
3030

31+
EXITCODE_BAD_TEST = 2
32+
EXITCODE_ENV_CHANGED = 3
33+
EXITCODE_NO_TESTS_RAN = 4
34+
EXITCODE_INTERRUPTED = 130
35+
3136
TEST_INTERRUPTED = textwrap.dedent("""
3237
from signal import SIGINT, raise_signal
3338
try:
@@ -497,7 +502,7 @@ def list_regex(line_format, tests):
497502
result.append('INTERRUPTED')
498503
if not any((good, result, failed, interrupted, skipped,
499504
env_changed, fail_env_changed)):
500-
result.append("NO TEST RUN")
505+
result.append("NO TESTS RAN")
501506
elif not result:
502507
result.append('SUCCESS')
503508
result = ', '.join(result)
@@ -707,7 +712,7 @@ def test_failing(self):
707712
test_failing = self.create_test('failing', code=code)
708713
tests = [test_ok, test_failing]
709714

710-
output = self.run_tests(*tests, exitcode=2)
715+
output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST)
711716
self.check_executed_tests(output, tests, failed=test_failing)
712717

713718
def test_resources(self):
@@ -748,13 +753,14 @@ def test_random(self):
748753
test = self.create_test('random', code)
749754

750755
# first run to get the output with the random seed
751-
output = self.run_tests('-r', test)
756+
output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN)
752757
randseed = self.parse_random_seed(output)
753758
match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
754759
test_random = int(match.group(1))
755760

756761
# try to reproduce with the random seed
757-
output = self.run_tests('-r', '--randseed=%s' % randseed, test)
762+
output = self.run_tests('-r', '--randseed=%s' % randseed, test,
763+
exitcode=EXITCODE_NO_TESTS_RAN)
758764
randseed2 = self.parse_random_seed(output)
759765
self.assertEqual(randseed2, randseed)
760766

@@ -813,7 +819,7 @@ def test_fromfile(self):
813819
def test_interrupted(self):
814820
code = TEST_INTERRUPTED
815821
test = self.create_test('sigint', code=code)
816-
output = self.run_tests(test, exitcode=130)
822+
output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED)
817823
self.check_executed_tests(output, test, omitted=test,
818824
interrupted=True)
819825

@@ -838,7 +844,7 @@ def test_slowest_interrupted(self):
838844
args = ("--slowest", "-j2", test)
839845
else:
840846
args = ("--slowest", test)
841-
output = self.run_tests(*args, exitcode=130)
847+
output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED)
842848
self.check_executed_tests(output, test,
843849
omitted=test, interrupted=True)
844850

@@ -878,7 +884,7 @@ def test_run(self):
878884
builtins.__dict__['RUN'] = 1
879885
""")
880886
test = self.create_test('forever', code=code)
881-
output = self.run_tests('--forever', test, exitcode=2)
887+
output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST)
882888
self.check_executed_tests(output, [test]*3, failed=test)
883889

884890
def check_leak(self, code, what):
@@ -887,7 +893,7 @@ def check_leak(self, code, what):
887893
filename = 'reflog.txt'
888894
self.addCleanup(os_helper.unlink, filename)
889895
output = self.run_tests('--huntrleaks', '3:3:', test,
890-
exitcode=2,
896+
exitcode=EXITCODE_BAD_TEST,
891897
stderr=subprocess.STDOUT)
892898
self.check_executed_tests(output, [test], failed=test)
893899

@@ -969,7 +975,7 @@ def test_crashed(self):
969975
crash_test = self.create_test(name="crash", code=code)
970976

971977
tests = [crash_test]
972-
output = self.run_tests("-j2", *tests, exitcode=2)
978+
output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST)
973979
self.check_executed_tests(output, tests, failed=crash_test,
974980
randomize=True)
975981

@@ -1069,7 +1075,8 @@ def test_env_changed(self):
10691075
self.check_executed_tests(output, [testname], env_changed=testname)
10701076

10711077
# fail with --fail-env-changed
1072-
output = self.run_tests("--fail-env-changed", testname, exitcode=3)
1078+
output = self.run_tests("--fail-env-changed", testname,
1079+
exitcode=EXITCODE_ENV_CHANGED)
10731080
self.check_executed_tests(output, [testname], env_changed=testname,
10741081
fail_env_changed=True)
10751082

@@ -1088,7 +1095,7 @@ def test_fail_always(self):
10881095
""")
10891096
testname = self.create_test(code=code)
10901097

1091-
output = self.run_tests("-w", testname, exitcode=2)
1098+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
10921099
self.check_executed_tests(output, [testname],
10931100
failed=testname, rerun={testname: "test_fail_always"})
10941101

@@ -1123,7 +1130,8 @@ def test_bug(self):
11231130
""")
11241131
testname = self.create_test(code=code)
11251132

1126-
output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0)
1133+
output = self.run_tests(testname, "-m", "nosuchtest",
1134+
exitcode=EXITCODE_NO_TESTS_RAN)
11271135
self.check_executed_tests(output, [testname], no_test_ran=testname)
11281136

11291137
def test_no_tests_ran_skip(self):
@@ -1136,7 +1144,7 @@ def test_skipped(self):
11361144
""")
11371145
testname = self.create_test(code=code)
11381146

1139-
output = self.run_tests(testname, exitcode=0)
1147+
output = self.run_tests(testname)
11401148
self.check_executed_tests(output, [testname])
11411149

11421150
def test_no_tests_ran_multiple_tests_nonexistent(self):
@@ -1150,7 +1158,8 @@ def test_bug(self):
11501158
testname = self.create_test(code=code)
11511159
testname2 = self.create_test(code=code)
11521160

1153-
output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0)
1161+
output = self.run_tests(testname, testname2, "-m", "nosuchtest",
1162+
exitcode=EXITCODE_NO_TESTS_RAN)
11541163
self.check_executed_tests(output, [testname, testname2],
11551164
no_test_ran=[testname, testname2])
11561165

@@ -1198,7 +1207,8 @@ def test_garbage(self):
11981207
""")
11991208
testname = self.create_test(code=code)
12001209

1201-
output = self.run_tests("--fail-env-changed", testname, exitcode=3)
1210+
output = self.run_tests("--fail-env-changed", testname,
1211+
exitcode=EXITCODE_ENV_CHANGED)
12021212
self.check_executed_tests(output, [testname],
12031213
env_changed=[testname],
12041214
fail_env_changed=True)
@@ -1224,7 +1234,8 @@ def test_sleep(self):
12241234
""")
12251235
testname = self.create_test(code=code)
12261236

1227-
output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2)
1237+
output = self.run_tests("-j2", "--timeout=1.0", testname,
1238+
exitcode=EXITCODE_BAD_TEST)
12281239
self.check_executed_tests(output, [testname],
12291240
failed=testname)
12301241
self.assertRegex(output,
@@ -1256,7 +1267,8 @@ def test_unraisable_exc(self):
12561267
""")
12571268
testname = self.create_test(code=code)
12581269

1259-
output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
1270+
output = self.run_tests("--fail-env-changed", "-v", testname,
1271+
exitcode=EXITCODE_ENV_CHANGED)
12601272
self.check_executed_tests(output, [testname],
12611273
env_changed=[testname],
12621274
fail_env_changed=True)
@@ -1287,7 +1299,8 @@ def test_threading_excepthook(self):
12871299
""")
12881300
testname = self.create_test(code=code)
12891301

1290-
output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
1302+
output = self.run_tests("--fail-env-changed", "-v", testname,
1303+
exitcode=EXITCODE_ENV_CHANGED)
12911304
self.check_executed_tests(output, [testname],
12921305
env_changed=[testname],
12931306
fail_env_changed=True)
@@ -1328,7 +1341,7 @@ def test_print_warning(self):
13281341
for option in ("-v", "-W"):
13291342
with self.subTest(option=option):
13301343
cmd = ["--fail-env-changed", option, testname]
1331-
output = self.run_tests(*cmd, exitcode=3)
1344+
output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED)
13321345
self.check_executed_tests(output, [testname],
13331346
env_changed=[testname],
13341347
fail_env_changed=True)
@@ -1373,7 +1386,8 @@ def test_leak_tmp_file(self):
13731386
""")
13741387
testnames = [self.create_test(code=code) for _ in range(3)]
13751388

1376-
output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, exitcode=3)
1389+
output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames,
1390+
exitcode=EXITCODE_ENV_CHANGED)
13771391
self.check_executed_tests(output, testnames,
13781392
env_changed=testnames,
13791393
fail_env_changed=True,

‎Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
The Python test suite now fails wit exit code 4 if no tests ran. It should
2+
help detecting typos in test names and test methods.

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL