FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-98627: Add an Optional Check for Extension Module Subinterpreter C… · python/cpython@89ac665 · GitHub

  1. python
  2. cpython

Repository navigation

Commit 89ac665

Browse files
ericsnowcurrently
authored
gh-98627: Add an Optional Check for Extension Module Subinterpreter Compatibility (gh-99040)
Enforcing (optionally) the restriction set by PEP 489 makes sense. Furthermore, this sets the stage for a potential restriction related to a per-interpreter GIL. This change includes the following: * add tests for extension module subinterpreter compatibility * add _PyInterpreterConfig.check_multi_interp_extensions * add Py_RTFLAGS_MULTI_INTERP_EXTENSIONS * add _PyImport_CheckSubinterpIncompatibleExtensionAllowed() * fail iff the module does not implement multi-phase init and the current interpreter is configured to check #98627
1 parent 3dea4ba commit 89ac665

15 files changed

Lines changed: 557 additions & 19 deletions

File tree

‎Include/cpython/initconfig.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ typedef struct {
248248
int allow_exec;
249249
int allow_threads;
250250
int allow_daemon_threads;
251+
int check_multi_interp_extensions;
251252
} _PyInterpreterConfig;
252253

253254
#define _PyInterpreterConfig_INIT \
@@ -256,6 +257,7 @@ typedef struct {
256257
.allow_exec = 0, \
257258
.allow_threads = 1, \
258259
.allow_daemon_threads = 0, \
260+
.check_multi_interp_extensions = 1, \
259261
}
260262

261263
#define _PyInterpreterConfig_LEGACY_INIT \
@@ -264,6 +266,7 @@ typedef struct {
264266
.allow_exec = 1, \
265267
.allow_threads = 1, \
266268
.allow_daemon_threads = 1, \
269+
.check_multi_interp_extensions = 0, \
267270
}
268271

269272
/* --- Helper functions --------------------------------------- */

‎Include/cpython/pystate.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ is available in a given context. For example, forking the process
1111
might not be allowed in the current interpreter (i.e. os.fork() would fail).
1212
*/
1313

14+
/* Set if import should check a module for subinterpreter support. */
15+
#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8)
16+
1417
/* Set if threads are allowed. */
1518
#define Py_RTFLAGS_THREADS (1UL << 10)
1619

‎Include/internal/pycore_import.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ struct _import_state {
6464
/* override for config->use_frozen_modules (for tests)
6565
(-1: "off", 1: "on", 0: no override) */
6666
int override_frozen_modules;
67+
int override_multi_interp_extensions_check;
6768
#ifdef HAVE_DLOPEN
6869
int dlopenflags;
6970
#endif
@@ -153,6 +154,10 @@ PyAPI_DATA(const struct _frozen *) _PyImport_FrozenStdlib;
153154
PyAPI_DATA(const struct _frozen *) _PyImport_FrozenTest;
154155
extern const struct _module_alias * _PyImport_FrozenAliases;
155156

157+
PyAPI_FUNC(int) _PyImport_CheckSubinterpIncompatibleExtensionAllowed(
158+
const char *name);
159+
160+
156161
// for testing
157162
PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename);
158163

‎Lib/test/support/import_helper.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,24 @@ def frozen_modules(enabled=True):
105105
_imp._override_frozen_modules_for_tests(0)
106106

107107

108+
@contextlib.contextmanager
109+
def multi_interp_extensions_check(enabled=True):
110+
"""Force legacy modules to be allowed in subinterpreters (or not).
111+
112+
("legacy" == single-phase init)
113+
114+
This only applies to modules that haven't been imported yet.
115+
It overrides the PyInterpreterConfig.check_multi_interp_extensions
116+
setting (see support.run_in_subinterp_with_config() and
117+
_xxsubinterpreters.create()).
118+
"""
119+
old = _imp._override_multi_interp_extensions_check(1 if enabled else -1)
120+
try:
121+
yield
122+
finally:
123+
_imp._override_multi_interp_extensions_check(old)
124+
125+
108126
def import_fresh_module(name, fresh=(), blocked=(), *,
109127
deprecated=False,
110128
usefrozen=False,

‎Lib/test/test_capi/check_config.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# This script is used by test_misc.
2+
3+
import _imp
4+
import _testinternalcapi
5+
import json
6+
import os
7+
import sys
8+
9+
10+
def import_singlephase():
11+
assert '_testsinglephase' not in sys.modules
12+
try:
13+
import _testsinglephase
14+
except ImportError:
15+
sys.modules.pop('_testsinglephase')
16+
return False
17+
else:
18+
del sys.modules['_testsinglephase']
19+
return True
20+
21+
22+
def check_singlephase(override):
23+
# Check using the default setting.
24+
settings_initial = _testinternalcapi.get_interp_settings()
25+
allowed_initial = import_singlephase()
26+
assert(_testinternalcapi.get_interp_settings() == settings_initial)
27+
28+
# Apply the override and check.
29+
override_initial = _imp._override_multi_interp_extensions_check(override)
30+
settings_after = _testinternalcapi.get_interp_settings()
31+
allowed_after = import_singlephase()
32+
33+
# Apply the override again and check.
34+
noop = {}
35+
override_after = _imp._override_multi_interp_extensions_check(override)
36+
settings_noop = _testinternalcapi.get_interp_settings()
37+
if settings_noop != settings_after:
38+
noop['settings_noop'] = settings_noop
39+
allowed_noop = import_singlephase()
40+
if allowed_noop != allowed_after:
41+
noop['allowed_noop'] = allowed_noop
42+
43+
# Restore the original setting and check.
44+
override_noop = _imp._override_multi_interp_extensions_check(override_initial)
45+
if override_noop != override_after:
46+
noop['override_noop'] = override_noop
47+
settings_restored = _testinternalcapi.get_interp_settings()
48+
allowed_restored = import_singlephase()
49+
50+
# Restore the original setting again.
51+
override_restored = _imp._override_multi_interp_extensions_check(override_initial)
52+
assert(_testinternalcapi.get_interp_settings() == settings_restored)
53+
54+
return dict({
55+
'requested': override,
56+
'override__initial': override_initial,
57+
'override_after': override_after,
58+
'override_restored': override_restored,
59+
'settings__initial': settings_initial,
60+
'settings_after': settings_after,
61+
'settings_restored': settings_restored,
62+
'allowed__initial': allowed_initial,
63+
'allowed_after': allowed_after,
64+
'allowed_restored': allowed_restored,
65+
}, **noop)
66+
67+
68+
def run_singlephase_check(override, outfd):
69+
with os.fdopen(outfd, 'w') as outfile:
70+
sys.stdout = outfile
71+
sys.stderr = outfile
72+
try:
73+
results = check_singlephase(override)
74+
json.dump(results, outfile)
75+
finally:
76+
sys.stdout = sys.__stdout__
77+
sys.stderr = sys.__stderr__

‎Lib/test/test_capi/test_misc.py‎

Lines changed: 93 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@
3131
import _testmultiphase
3232
except ImportError:
3333
_testmultiphase = None
34+
try:
35+
import _testsinglephase
36+
except ImportError:
37+
_testsinglephase = None
3438

3539
# Skip this test if the _testcapi module isn't available.
3640
_testcapi = import_helper.import_module('_testcapi')
@@ -1297,17 +1301,20 @@ def test_configured_settings(self):
12971301
"""
12981302
import json
12991303

1304+
EXTENSIONS = 1<<8
13001305
THREADS = 1<<10
13011306
DAEMON_THREADS = 1<<11
13021307
FORK = 1<<15
13031308
EXEC = 1<<16
13041309

1305-
features = ['fork', 'exec', 'threads', 'daemon_threads']
1310+
features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions']
13061311
kwlist = [f'allow_{n}' for n in features]
1312+
kwlist[-1] = 'check_multi_interp_extensions'
13071313
for config, expected in {
1308-
(True, True, True, True): FORK | EXEC | THREADS | DAEMON_THREADS,
1309-
(False, False, False, False): 0,
1310-
(False, False, True, False): THREADS,
1314+
(True, True, True, True, True):
1315+
FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
1316+
(False, False, False, False, False): 0,
1317+
(False, False, True, False, True): THREADS | EXTENSIONS,
13111318
}.items():
13121319
kwargs = dict(zip(kwlist, config))
13131320
expected = {
@@ -1322,12 +1329,93 @@ def test_configured_settings(self):
13221329
json.dump(settings, stdin)
13231330
''')
13241331
with os.fdopen(r) as stdout:
1325-
support.run_in_subinterp_with_config(script, **kwargs)
1332+
ret = support.run_in_subinterp_with_config(script, **kwargs)
1333+
self.assertEqual(ret, 0)
13261334
out = stdout.read()
13271335
settings = json.loads(out)
13281336

13291337
self.assertEqual(settings, expected)
13301338

1339+
@unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
1340+
@unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
1341+
def test_overridden_setting_extensions_subinterp_check(self):
1342+
"""
1343+
PyInterpreterConfig.check_multi_interp_extensions can be overridden
1344+
with PyInterpreterState.override_multi_interp_extensions_check.
1345+
This verifies that the override works but does not modify
1346+
the underlying setting.
1347+
"""
1348+
import json
1349+
1350+
EXTENSIONS = 1<<8
1351+
THREADS = 1<<10
1352+
DAEMON_THREADS = 1<<11
1353+
FORK = 1<<15
1354+
EXEC = 1<<16
1355+
BASE_FLAGS = FORK | EXEC | THREADS | DAEMON_THREADS
1356+
base_kwargs = {
1357+
'allow_fork': True,
1358+
'allow_exec': True,
1359+
'allow_threads': True,
1360+
'allow_daemon_threads': True,
1361+
}
1362+
1363+
def check(enabled, override):
1364+
kwargs = dict(
1365+
base_kwargs,
1366+
check_multi_interp_extensions=enabled,
1367+
)
1368+
flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS
1369+
settings = {
1370+
'feature_flags': flags,
1371+
}
1372+
1373+
expected = {
1374+
'requested': override,
1375+
'override__initial': 0,
1376+
'override_after': override,
1377+
'override_restored': 0,
1378+
# The override should not affect the config or settings.
1379+
'settings__initial': settings,
1380+
'settings_after': settings,
1381+
'settings_restored': settings,
1382+
# These are the most likely values to be wrong.
1383+
'allowed__initial': not enabled,
1384+
'allowed_after': not ((override > 0) if override else enabled),
1385+
'allowed_restored': not enabled,
1386+
}
1387+
1388+
r, w = os.pipe()
1389+
script = textwrap.dedent(f'''
1390+
from test.test_capi.check_config import run_singlephase_check
1391+
run_singlephase_check({override}, {w})
1392+
''')
1393+
with os.fdopen(r) as stdout:
1394+
ret = support.run_in_subinterp_with_config(script, **kwargs)
1395+
self.assertEqual(ret, 0)
1396+
out = stdout.read()
1397+
results = json.loads(out)
1398+
1399+
self.assertEqual(results, expected)
1400+
1401+
self.maxDiff = None
1402+
1403+
# setting: check disabled
1404+
with self.subTest('config: check disabled; override: disabled'):
1405+
check(False, -1)
1406+
with self.subTest('config: check disabled; override: use config'):
1407+
check(False, 0)
1408+
with self.subTest('config: check disabled; override: enabled'):
1409+
check(False, 1)
1410+
1411+
# setting: check enabled
1412+
with self.subTest('config: check enabled; override: disabled'):
1413+
check(True, -1)
1414+
with self.subTest('config: check enabled; override: use config'):
1415+
check(True, 0)
1416+
with self.subTest('config: check enabled; override: enabled'):
1417+
check(True, 1)
1418+
13311419
def test_mutate_exception(self):
13321420
"""
13331421
Exceptions saved in global module state get shared between

‎Lib/test/test_embed.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1656,13 +1656,15 @@ def test_init_use_frozen_modules(self):
16561656
api=API_PYTHON, env=env)
16571657

16581658
def test_init_main_interpreter_settings(self):
1659+
EXTENSIONS = 1<<8
16591660
THREADS = 1<<10
16601661
DAEMON_THREADS = 1<<11
16611662
FORK = 1<<15
16621663
EXEC = 1<<16
16631664
expected = {
16641665
# All optional features should be enabled.
1665-
'feature_flags': FORK | EXEC | THREADS | DAEMON_THREADS,
1666+
'feature_flags':
1667+
FORK | EXEC | THREADS | DAEMON_THREADS,
16661668
}
16671669
out, err = self.run_embedded_interpreter(
16681670
'test_init_main_interpreter_settings',

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL