FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

bpo-45020: Identify which frozen modules are actually aliases. (gh-28… · python/cpython@08285d5 · GitHub

/ cpython Public

Commit 08285d5

Browse files
ericsnowcurrently
authored
bpo-45020: Identify which frozen modules are actually aliases. (gh-28655)
In the list of generated frozen modules at the top of Tools/scripts/freeze_modules.py, you will find that some of the modules have a different name than the module (or .py file) that is actually frozen. Let's call each case an "alias". Aliases do not come into play until we get to the (generated) list of modules in Python/frozen.c. (The tool for freezing modules, Programs/_freeze_module, is only concerned with the source file, not the module it will be used for.) Knowledge of which frozen modules are aliases (and the identity of the original module) normally isn't important. However, this information is valuable when we go to set __file__ on frozen stdlib modules. This change updates Tools/scripts/freeze_modules.py to map aliases to the original module name (or None if not a stdlib module) in Python/frozen.c. We also add a helper function in Python/import.c to look up a frozen module's alias and add the result of that function to the frozen info returned from find_frozen(). https://bugs.python.org/issue45020
1 parent 4444291 commit 08285d5

10 files changed

Lines changed: 225 additions & 37 deletions

File tree

‎Include/internal/pycore_import.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@ extern PyStatus _PyImport_ReInitLock(void);
1010
#endif
1111
extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate);
1212

13+
struct _module_alias {
14+
const char *name; /* ASCII encoded string */
15+
const char *orig; /* ASCII encoded string */
16+
};
17+
18+
extern const struct _module_alias * _PyImport_FrozenAliases;
19+
1320
#ifdef __cplusplus
1421
}
1522
#endif

‎Lib/importlib/_bootstrap.py‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -824,16 +824,39 @@ def module_repr(m):
824824
"slated for removal in Python 3.12", DeprecationWarning)
825825
return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)
826826

827+
@classmethod
828+
def _setup_module(cls, module):
829+
assert not hasattr(module, '__file__'), module.__file__
830+
ispkg = hasattr(module, '__path__')
831+
assert not ispkg or not module.__path__, module.__path__
832+
spec = module.__spec__
833+
assert not ispkg or not spec.submodule_search_locations
834+
835+
if spec.loader_state is None:
836+
spec.loader_state = type(sys.implementation)(
837+
data=None,
838+
origname=None,
839+
)
840+
elif not hasattr(spec.loader_state, 'data'):
841+
spec.loader_state.data = None
842+
if not getattr(spec.loader_state, 'origname', None):
843+
origname = vars(module).pop('__origname__', None)
844+
assert origname, 'see PyImport_ImportFrozenModuleObject()'
845+
spec.loader_state.origname = origname
846+
827847
@classmethod
828848
def find_spec(cls, fullname, path=None, target=None):
829849
info = _call_with_frames_removed(_imp.find_frozen, fullname)
830850
if info is None:
831851
return None
832-
data, ispkg = info
852+
data, ispkg, origname = info
833853
spec = spec_from_loader(fullname, cls,
834854
origin=cls._ORIGIN,
835855
is_package=ispkg)
836-
spec.loader_state = data
856+
spec.loader_state = type(sys.implementation)(
857+
data=data,
858+
origname=origname,
859+
)
837860
return spec
838861

839862
@classmethod
@@ -857,7 +880,7 @@ def exec_module(module):
857880
spec = module.__spec__
858881
name = spec.name
859882
try:
860-
data = spec.loader_state
883+
data = spec.loader_state.data
861884
except AttributeError:
862885
if not _imp.is_frozen(name):
863886
raise ImportError('{!r} is not a frozen module'.format(name),
@@ -868,7 +891,7 @@ def exec_module(module):
868891
# Note that if this method is called again (e.g. by
869892
# importlib.reload()) then _imp.get_frozen_object() will notice
870893
# no data was provided and will look it up.
871-
spec.loader_state = None
894+
spec.loader_state.data = None
872895
code = _call_with_frames_removed(_imp.get_frozen_object, name, data)
873896
exec(code, module.__dict__)
874897

@@ -1220,6 +1243,8 @@ def _setup(sys_module, _imp_module):
12201243
continue
12211244
spec = _spec_from_module(module, loader)
12221245
_init_module_attrs(spec, module)
1246+
if loader is FrozenImporter:
1247+
loader._setup_module(module)
12231248

12241249
# Directly load built-in modules needed during bootstrap.
12251250
self_module = sys.modules[__name__]

‎Lib/test/test_importlib/frozen/test_finder.py‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,15 @@
99
import unittest
1010
import warnings
1111

12-
from test.support import import_helper, REPO_ROOT
12+
from test.support import import_helper, REPO_ROOT, STDLIB_DIR
13+
14+
15+
def resolve_stdlib_file(name, ispkg=False):
16+
assert name
17+
if ispkg:
18+
return os.path.join(STDLIB_DIR, *name.split('.'), '__init__.py')
19+
else:
20+
return os.path.join(STDLIB_DIR, *name.split('.')) + '.py'
1321

1422

1523
class FindSpecTests(abc.FinderTests):
@@ -32,16 +40,30 @@ def check_basic(self, spec, name, ispkg=False):
3240
self.assertIsNone(spec.submodule_search_locations)
3341
self.assertIsNotNone(spec.loader_state)
3442

35-
def check_data(self, spec):
43+
def check_loader_state(self, spec, origname=None, filename=None):
44+
if not filename:
45+
if not origname:
46+
origname = spec.name
47+
48+
actual = dict(vars(spec.loader_state))
49+
50+
# Check the code object used to import the frozen module.
51+
# We can't compare the marshaled data directly because
52+
# marshal.dumps() would mark "expected" (below) as a ref,
53+
# which slightly changes the output.
54+
# (See https://bugs.python.org/issue34093.)
55+
data = actual.pop('data')
3656
with import_helper.frozen_modules():
3757
expected = _imp.get_frozen_object(spec.name)
38-
data = spec.loader_state
39-
# We can't compare the marshaled data directly because
40-
# marshal.dumps() would mark "expected" as a ref, which slightly
41-
# changes the output. (See https://bugs.python.org/issue34093.)
4258
code = marshal.loads(data)
4359
self.assertEqual(code, expected)
4460

61+
# Check the rest of spec.loader_state.
62+
expected = dict(
63+
origname=origname,
64+
)
65+
self.assertDictEqual(actual, expected)
66+
4567
def check_search_locations(self, spec):
4668
# Frozen packages do not have any path entries.
4769
# (See https://bugs.python.org/issue21736.)
@@ -58,7 +80,7 @@ def test_module(self):
5880
with self.subTest(f'{name} -> {name}'):
5981
spec = self.find(name)
6082
self.check_basic(spec, name)
61-
self.check_data(spec)
83+
self.check_loader_state(spec)
6284
modules = {
6385
'__hello_alias__': '__hello__',
6486
'_frozen_importlib': 'importlib._bootstrap',
@@ -67,46 +89,50 @@ def test_module(self):
6789
with self.subTest(f'{name} -> {origname}'):
6890
spec = self.find(name)
6991
self.check_basic(spec, name)
70-
self.check_data(spec)
92+
self.check_loader_state(spec, origname)
7193
modules = [
7294
'__phello__.__init__',
7395
'__phello__.ham.__init__',
7496
]
7597
for name in modules:
76-
origname = name.rpartition('.')[0]
98+
origname = '<' + name.rpartition('.')[0]
99+
filename = resolve_stdlib_file(name)
77100
with self.subTest(f'{name} -> {origname}'):
78101
spec = self.find(name)
79102
self.check_basic(spec, name)
80-
self.check_data(spec)
103+
self.check_loader_state(spec, origname, filename)
81104
modules = {
82105
'__hello_only__': ('Tools', 'freeze', 'flag.py'),
83106
}
84107
for name, path in modules.items():
108+
origname = None
85109
filename = os.path.join(REPO_ROOT, *path)
86110
with self.subTest(f'{name} -> {filename}'):
87111
spec = self.find(name)
88112
self.check_basic(spec, name)
89-
self.check_data(spec)
113+
self.check_loader_state(spec, origname, filename)
90114

91115
def test_package(self):
92116
packages = [
93117
'__phello__',
94118
'__phello__.ham',
95119
]
96120
for name in packages:
121+
filename = resolve_stdlib_file(name, ispkg=True)
97122
with self.subTest(f'{name} -> {name}'):
98123
spec = self.find(name)
99124
self.check_basic(spec, name, ispkg=True)
100-
self.check_data(spec)
125+
self.check_loader_state(spec, name, filename)
101126
self.check_search_locations(spec)
102127
packages = {
103128
'__phello_alias__': '__hello__',
104129
}
105130
for name, origname in packages.items():
131+
filename = resolve_stdlib_file(origname, ispkg=False)
106132
with self.subTest(f'{name} -> {origname}'):
107133
spec = self.find(name)
108134
self.check_basic(spec, name, ispkg=True)
109-
self.check_data(spec)
135+
self.check_loader_state(spec, origname, filename)
110136
self.check_search_locations(spec)
111137

112138
# These are covered by test_module() and test_package().

‎Lib/test/test_importlib/frozen/test_loader.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,17 +32,19 @@ def fresh(name, *, oldapi=False):
3232

3333
class ExecModuleTests(abc.LoaderTests):
3434

35-
def exec_module(self, name):
35+
def exec_module(self, name, origname=None):
3636
with import_helper.frozen_modules():
3737
is_package = self.machinery.FrozenImporter.is_package(name)
3838
code = _imp.get_frozen_object(name)
39-
data = marshal.dumps(code)
4039
spec = self.machinery.ModuleSpec(
4140
name,
4241
self.machinery.FrozenImporter,
4342
origin='frozen',
4443
is_package=is_package,
45-
loader_state=data,
44+
loader_state=types.SimpleNamespace(
45+
data=marshal.dumps(code),
46+
origname=origname or name,
47+
),
4648
)
4749
module = types.ModuleType(name)
4850
module.__spec__ = spec
@@ -66,7 +68,8 @@ def test_module(self):
6668
self.assertEqual(getattr(module, attr), value)
6769
self.assertEqual(output, 'Hello world!\n')
6870
self.assertTrue(hasattr(module, '__spec__'))
69-
self.assertIsNone(module.__spec__.loader_state)
71+
self.assertIsNone(module.__spec__.loader_state.data)
72+
self.assertEqual(module.__spec__.loader_state.origname, name)
7073

7174
def test_package(self):
7275
name = '__phello__'
@@ -79,7 +82,8 @@ def test_package(self):
7982
name=name, attr=attr, given=attr_value,
8083
expected=value))
8184
self.assertEqual(output, 'Hello world!\n')
82-
self.assertIsNone(module.__spec__.loader_state)
85+
self.assertIsNone(module.__spec__.loader_state.data)
86+
self.assertEqual(module.__spec__.loader_state.origname, name)
8387

8488
def test_lacking_parent(self):
8589
name = '__phello__.spam'

‎Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-06-54.bpo-45020.Cj5VQN.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
For frozen stdlib modules, record the original module name as
2+
``module.__spec__.loader_state.origname``. If the value is different than
3+
``module.__spec__.name`` then the module was defined as an alias in
4+
Tools/scripts/freeze_modules.py. If it is ``None`` then the module comes
5+
from a source file outside the stdlib.

‎Programs/_freeze_module.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
#include <Python.h>
1111
#include <marshal.h>
12+
#include <pycore_import.h>
1213

1314
#include <stdio.h>
1415
#include <sys/types.h>
@@ -24,8 +25,12 @@
2425
static const struct _frozen _PyImport_FrozenModules[] = {
2526
{0, 0, 0} /* sentinel */
2627
};
28+
static const struct _module_alias aliases[] = {
29+
{0, 0} /* sentinel */
30+
};
2731

2832
const struct _frozen *PyImport_FrozenModules;
33+
const struct _module_alias *_PyImport_FrozenAliases;
2934

3035
static const char header[] =
3136
"/* Auto-generated by Programs/_freeze_module.c */";
@@ -183,6 +188,7 @@ main(int argc, char *argv[])
183188
const char *name, *inpath, *outpath;
184189

185190
PyImport_FrozenModules = _PyImport_FrozenModules;
191+
_PyImport_FrozenAliases = aliases;
186192

187193
if (argc != 4) {
188194
fprintf(stderr, "need to specify the name, input and output paths\n");

‎Python/clinic/import.c.h‎

Lines changed: 5 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/frozen.c‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
and __phello__.spam. Loading any will print some famous words... */
3737

3838
#include "Python.h"
39+
#include "pycore_import.h"
3940

4041
/* Includes for frozen modules: */
4142
#include "frozen_modules/importlib._bootstrap.h"
@@ -102,9 +103,24 @@ static const struct _frozen _PyImport_FrozenModules[] = {
102103
{"__phello__.spam", _Py_M____phello___spam,
103104
(int)sizeof(_Py_M____phello___spam)},
104105
{"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)},
105-
{0, 0, 0} /* sentinel */
106+
{0, 0, 0} /* modules sentinel */
106107
};
107108

109+
static const struct _module_alias aliases[] = {
110+
{"_frozen_importlib", "importlib._bootstrap"},
111+
{"_frozen_importlib_external", "importlib._bootstrap_external"},
112+
{"os.path", "posixpath"},
113+
{"__hello_alias__", "__hello__"},
114+
{"__phello_alias__", "__hello__"},
115+
{"__phello_alias__.spam", "__hello__"},
116+
{"__phello__.__init__", "<__phello__"},
117+
{"__phello__.ham.__init__", "<__phello__.ham"},
118+
{"__hello_only__", NULL},
119+
{0, 0} /* aliases sentinel */
120+
};
121+
const struct _module_alias *_PyImport_FrozenAliases = aliases;
122+
123+
108124
/* Embedding apps may change this pointer to point to their favorite
109125
collection of frozen modules: */
110126

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL