FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

[3.11] bpo-46523: fix tests rerun when `setUp[Class|Module]` fails (G… · python/cpython@ecb09a8 · GitHub

  1. python
  2. cpython

Repository navigation

Commit ecb09a8

Browse files
ambv
and
sobolevn
authored
[3.11] bpo-46523: fix tests rerun when setUp[Class|Module] fails (GH-30895) (GH-103342)
(cherry picked from commit 9953860) Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
1 parent 8740fd8 commit ecb09a8

3 files changed

Lines changed: 194 additions & 3 deletions

File tree

‎Lib/test/libregrtest/main.py‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,19 @@
2828
# Must be smaller than buildbot "1200 seconds without output" limit.
2929
EXIT_TIMEOUT = 120.0
3030

31+
# gh-90681: When rerunning tests, we might need to rerun the whole
32+
# class or module suite if some its life-cycle hooks fail.
33+
# Test level hooks are not affected.
34+
_TEST_LIFECYCLE_HOOKS = frozenset((
35+
'setUpClass', 'tearDownClass',
36+
'setUpModule', 'tearDownModule',
37+
))
38+
39+
EXITCODE_BAD_TEST = 2
40+
EXITCODE_INTERRUPTED = 130
41+
EXITCODE_ENV_CHANGED = 3
42+
EXITCODE_NO_TESTS_RAN = 4
43+
3144

3245
class Regrtest:
3346
"""Execute a test suite.
@@ -331,8 +344,12 @@ def rerun_failed_tests(self):
331344

332345
errors = result.errors or []
333346
failures = result.failures or []
334-
error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
335-
failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
347+
error_names = [
348+
self.normalize_test_name(test_full_name, is_error=True)
349+
for (test_full_name, *_) in errors]
350+
failure_names = [
351+
self.normalize_test_name(test_full_name)
352+
for (test_full_name, *_) in failures]
336353
self.ns.verbose = True
337354
orig_match_tests = self.ns.match_tests
338355
if errors or failures:
@@ -358,6 +375,21 @@ def rerun_failed_tests(self):
358375

359376
self.display_result()
360377

378+
def normalize_test_name(self, test_full_name, *, is_error=False):
379+
short_name = test_full_name.split(" ")[0]
380+
if is_error and short_name in _TEST_LIFECYCLE_HOOKS:
381+
# This means that we have a failure in a life-cycle hook,
382+
# we need to rerun the whole module or class suite.
383+
# Basically the error looks like this:
384+
# ERROR: setUpClass (test.test_reg_ex.RegTest)
385+
# or
386+
# ERROR: setUpModule (test.test_reg_ex)
387+
# So, we need to parse the class / module name.
388+
lpar = test_full_name.index('(')
389+
rpar = test_full_name.index(')')
390+
return test_full_name[lpar + 1: rpar].split('.')[-1]
391+
return short_name
392+
361393
def display_result(self):
362394
# If running the test suite for PGO then no one cares about results.
363395
if self.ns.pgo:

‎Lib/test/support/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1097,7 +1097,7 @@ def _run_suite(suite):
10971097
if junit_xml_list is not None:
10981098
junit_xml_list.append(result.get_xml_element())
10991099

1100-
if not result.testsRun and not result.skipped:
1100+
if not result.testsRun and not result.skipped and not result.errors:
11011101
raise TestDidNotRun
11021102
if not result.wasSuccessful():
11031103
if len(result.errors) == 1 and not result.failures:

‎Lib/test/test_regrtest.py‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@
3030
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
3131
LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
3232

33+
EXITCODE_BAD_TEST = 2
34+
EXITCODE_ENV_CHANGED = 3
35+
EXITCODE_NO_TESTS_RAN = 4
36+
EXITCODE_INTERRUPTED = 130
37+
3338
TEST_INTERRUPTED = textwrap.dedent("""
3439
from signal import SIGINT, raise_signal
3540
try:
@@ -1115,6 +1120,160 @@ def test_fail_once(self):
11151120
self.check_executed_tests(output, [testname],
11161121
rerun={testname: "test_fail_once"})
11171122

1123+
def test_rerun_setup_class_hook_failure(self):
1124+
# FAILURE then FAILURE
1125+
code = textwrap.dedent("""
1126+
import unittest
1127+
1128+
class ExampleTests(unittest.TestCase):
1129+
@classmethod
1130+
def setUpClass(self):
1131+
raise RuntimeError('Fail')
1132+
1133+
def test_success(self):
1134+
return
1135+
""")
1136+
testname = self.create_test(code=code)
1137+
1138+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1139+
self.check_executed_tests(output, testname,
1140+
failed=[testname],
1141+
rerun={testname: "ExampleTests"})
1142+
1143+
def test_rerun_teardown_class_hook_failure(self):
1144+
# FAILURE then FAILURE
1145+
code = textwrap.dedent("""
1146+
import unittest
1147+
1148+
class ExampleTests(unittest.TestCase):
1149+
@classmethod
1150+
def tearDownClass(self):
1151+
raise RuntimeError('Fail')
1152+
1153+
def test_success(self):
1154+
return
1155+
""")
1156+
testname = self.create_test(code=code)
1157+
1158+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1159+
self.check_executed_tests(output, testname,
1160+
failed=[testname],
1161+
rerun={testname: "ExampleTests"})
1162+
1163+
def test_rerun_setup_module_hook_failure(self):
1164+
# FAILURE then FAILURE
1165+
code = textwrap.dedent("""
1166+
import unittest
1167+
1168+
def setUpModule():
1169+
raise RuntimeError('Fail')
1170+
1171+
class ExampleTests(unittest.TestCase):
1172+
def test_success(self):
1173+
return
1174+
""")
1175+
testname = self.create_test(code=code)
1176+
1177+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1178+
self.check_executed_tests(output, testname,
1179+
failed=[testname],
1180+
rerun={testname: testname})
1181+
1182+
def test_rerun_teardown_module_hook_failure(self):
1183+
# FAILURE then FAILURE
1184+
code = textwrap.dedent("""
1185+
import unittest
1186+
1187+
def tearDownModule():
1188+
raise RuntimeError('Fail')
1189+
1190+
class ExampleTests(unittest.TestCase):
1191+
def test_success(self):
1192+
return
1193+
""")
1194+
testname = self.create_test(code=code)
1195+
1196+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1197+
self.check_executed_tests(output, testname,
1198+
failed=[testname],
1199+
rerun={testname: testname})
1200+
1201+
def test_rerun_setup_hook_failure(self):
1202+
# FAILURE then FAILURE
1203+
code = textwrap.dedent("""
1204+
import unittest
1205+
1206+
class ExampleTests(unittest.TestCase):
1207+
def setUp(self):
1208+
raise RuntimeError('Fail')
1209+
1210+
def test_success(self):
1211+
return
1212+
""")
1213+
testname = self.create_test(code=code)
1214+
1215+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1216+
self.check_executed_tests(output, testname,
1217+
failed=[testname],
1218+
rerun={testname: "test_success"})
1219+
1220+
def test_rerun_teardown_hook_failure(self):
1221+
# FAILURE then FAILURE
1222+
code = textwrap.dedent("""
1223+
import unittest
1224+
1225+
class ExampleTests(unittest.TestCase):
1226+
def tearDown(self):
1227+
raise RuntimeError('Fail')
1228+
1229+
def test_success(self):
1230+
return
1231+
""")
1232+
testname = self.create_test(code=code)
1233+
1234+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1235+
self.check_executed_tests(output, testname,
1236+
failed=[testname],
1237+
rerun={testname: "test_success"})
1238+
1239+
def test_rerun_async_setup_hook_failure(self):
1240+
# FAILURE then FAILURE
1241+
code = textwrap.dedent("""
1242+
import unittest
1243+
1244+
class ExampleTests(unittest.IsolatedAsyncioTestCase):
1245+
async def asyncSetUp(self):
1246+
raise RuntimeError('Fail')
1247+
1248+
async def test_success(self):
1249+
return
1250+
""")
1251+
testname = self.create_test(code=code)
1252+
1253+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1254+
self.check_executed_tests(output, testname,
1255+
failed=[testname],
1256+
rerun={testname: "test_success"})
1257+
1258+
def test_rerun_async_teardown_hook_failure(self):
1259+
# FAILURE then FAILURE
1260+
code = textwrap.dedent("""
1261+
import unittest
1262+
1263+
class ExampleTests(unittest.IsolatedAsyncioTestCase):
1264+
async def asyncTearDown(self):
1265+
raise RuntimeError('Fail')
1266+
1267+
async def test_success(self):
1268+
return
1269+
""")
1270+
testname = self.create_test(code=code)
1271+
1272+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1273+
self.check_executed_tests(output, testname,
1274+
failed=[testname],
1275+
rerun={testname: "test_success"})
1276+
11181277
def test_no_tests_ran(self):
11191278
code = textwrap.dedent("""
11201279
import unittest

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL