| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 9b4e3d9 commit bf95ff9
9 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -256,8 +256,12 @@ Importing Modules | |||
| 256 | 256 | const char *name; | |
| 257 | 257 | const unsigned char *code; | |
| 258 | 258 | int size; | |
| 259 | + bool is_package; | ||
| 259 | 260 | }; | |
| 260 | 261 | ||
| 262 | + .. versionchanged:: 3.11 | ||
| 263 | + The new ``is_package`` field indicates whether the module is a package or not. | ||
| 264 | + This replaces setting the ``size`` field to a negative value. | ||
| 261 | 265 | ||
| 262 | 266 | .. c:var:: const struct _frozen* PyImport_FrozenModules | |
| 263 | 267 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -650,6 +650,11 @@ C API Changes | |||
| 650 | 650 | fields of the result from the exception instance (the ``value`` field). | |
| 651 | 651 | (Contributed by Irit Katriel in :issue:`45711`.) | |
| 652 | 652 | ||
| 653 | + * :c:struct:`_frozen` has a new ``is_package`` field to indicate whether | ||
| 654 | + or not the frozen module is a package. Previously, a negative value | ||
| 655 | + in the ``size`` field was the indicator. Now only non-negative values | ||
| 656 | + be used for ``size``. | ||
| 657 | + (Contributed by Kumar Aditya in :issue:`46608`.) | ||
| 653 | 658 | ||
| 654 | 659 | New Features | |
| 655 | 660 | ------------ | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -32,6 +32,7 @@ struct _frozen { | |||
| 32 | 32 | const char *name; /* ASCII encoded string */ | |
| 33 | 33 | const unsigned char *code; | |
| 34 | 34 | int size; | |
| 35 | + bool is_package; | ||
| 35 | 36 | PyObject *(*get_code)(void); | |
| 36 | 37 | }; | |
| 37 | 38 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -54,6 +54,7 @@ class struct_frozen(Structure): | |||
| 54 | 54 | _fields_ = [("name", c_char_p), | |
| 55 | 55 | ("code", POINTER(c_ubyte)), | |
| 56 | 56 | ("size", c_int), | |
| 57 | + ("is_package", c_bool), | ||
| 57 | 58 | ("get_code", POINTER(c_ubyte)), # Function ptr | |
| 58 | 59 | ] | |
| 59 | 60 | FrozenTable = POINTER(struct_frozen) | |
@@ -71,13 +72,14 @@ class struct_frozen(Structure): | |||
| 71 | 72 | modname = entry.name.decode("ascii") | |
| 72 | 73 | modules.append(modname) | |
| 73 | 74 | with self.subTest(modname): | |
| 74 | - # Do a sanity check on entry.size and entry.code. | ||
| 75 | - self.assertGreater(abs(entry.size), 10) | ||
| 76 | - self.assertTrue([entry.code[i] for i in range(abs(entry.size))]) | ||
| 75 | + if entry.size != 0: | ||
| 76 | + # Do a sanity check on entry.size and entry.code. | ||
| 77 | + self.assertGreater(abs(entry.size), 10) | ||
| 78 | + self.assertTrue([entry.code[i] for i in range(abs(entry.size))]) | ||
| 77 | 79 | # Check the module's package-ness. | |
| 78 | 80 | with import_helper.frozen_modules(): | |
| 79 | 81 | spec = importlib.util.find_spec(modname) | |
| 80 | - if entry.size < 0: | ||
| 82 | + if entry.is_package: | ||
| 81 | 83 | # It's a package. | |
| 82 | 84 | self.assertIsNotNone(spec.submodule_search_locations) | |
| 83 | 85 | else: | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,2 @@ | |||
| 1 | + Exclude marshalled-frozen data if deep-freezing to save 300 KB disk space. This includes adding | ||
| 2 | + a new ``is_package`` field to :c:struct:`_frozen`. Patch by Kumar Aditya. | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -39,29 +39,6 @@ | |||
| 39 | 39 | #include "pycore_import.h" | |
| 40 | 40 | ||
| 41 | 41 | /* Includes for frozen modules: */ | |
| 42 | - #include "frozen_modules/importlib._bootstrap.h" | ||
| 43 | - #include "frozen_modules/importlib._bootstrap_external.h" | ||
| 44 | - #include "frozen_modules/zipimport.h" | ||
| 45 | - #include "frozen_modules/abc.h" | ||
| 46 | - #include "frozen_modules/codecs.h" | ||
| 47 | - #include "frozen_modules/io.h" | ||
| 48 | - #include "frozen_modules/_collections_abc.h" | ||
| 49 | - #include "frozen_modules/_sitebuiltins.h" | ||
| 50 | - #include "frozen_modules/genericpath.h" | ||
| 51 | - #include "frozen_modules/ntpath.h" | ||
| 52 | - #include "frozen_modules/posixpath.h" | ||
| 53 | - #include "frozen_modules/os.h" | ||
| 54 | - #include "frozen_modules/site.h" | ||
| 55 | - #include "frozen_modules/stat.h" | ||
| 56 | - #include "frozen_modules/importlib.util.h" | ||
| 57 | - #include "frozen_modules/importlib.machinery.h" | ||
| 58 | - #include "frozen_modules/runpy.h" | ||
| 59 | - #include "frozen_modules/__hello__.h" | ||
| 60 | - #include "frozen_modules/__phello__.h" | ||
| 61 | - #include "frozen_modules/__phello__.ham.h" | ||
| 62 | - #include "frozen_modules/__phello__.ham.eggs.h" | ||
| 63 | - #include "frozen_modules/__phello__.spam.h" | ||
| 64 | - #include "frozen_modules/frozen_only.h" | ||
| 65 | 42 | /* End includes */ | |
| 66 | 43 | ||
| 67 | 44 | #define GET_CODE(name) _Py_get_##name##_toplevel | |
@@ -98,49 +75,47 @@ extern PyObject *_Py_get___phello___spam_toplevel(void); | |||
| 98 | 75 | extern PyObject *_Py_get_frozen_only_toplevel(void); | |
| 99 | 76 | /* End extern declarations */ | |
| 100 | 77 | ||
| 101 | - /* Note that a negative size indicates a package. */ | ||
| 102 | - | ||
| 103 | 78 | static const struct _frozen bootstrap_modules[] = { | |
| 104 | - {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), GET_CODE(importlib__bootstrap)}, | ||
| 105 | - {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), GET_CODE(importlib__bootstrap_external)}, | ||
| 106 | - {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), GET_CODE(zipimport)}, | ||
| 79 | + {"_frozen_importlib", NULL, 0, false, GET_CODE(importlib__bootstrap)}, | ||
| 80 | + {"_frozen_importlib_external", NULL, 0, false, GET_CODE(importlib__bootstrap_external)}, | ||
| 81 | + {"zipimport", NULL, 0, false, GET_CODE(zipimport)}, | ||
| 107 | 82 | {0, 0, 0} /* bootstrap sentinel */ | |
| 108 | 83 | }; | |
| 109 | 84 | static const struct _frozen stdlib_modules[] = { | |
| 110 | 85 | /* stdlib - startup, without site (python -S) */ | |
| 111 | - {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), GET_CODE(abc)}, | ||
| 112 | - {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), GET_CODE(codecs)}, | ||
| 113 | - {"io", _Py_M__io, (int)sizeof(_Py_M__io), GET_CODE(io)}, | ||
| 86 | + {"abc", NULL, 0, false, GET_CODE(abc)}, | ||
| 87 | + {"codecs", NULL, 0, false, GET_CODE(codecs)}, | ||
| 88 | + {"io", NULL, 0, false, GET_CODE(io)}, | ||
| 114 | 89 | ||
| 115 | 90 | /* stdlib - startup, with site */ | |
| 116 | - {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc), GET_CODE(_collections_abc)}, | ||
| 117 | - {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins), GET_CODE(_sitebuiltins)}, | ||
| 118 | - {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), GET_CODE(genericpath)}, | ||
| 119 | - {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), GET_CODE(ntpath)}, | ||
| 120 | - {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)}, | ||
| 121 | - {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)}, | ||
| 122 | - {"os", _Py_M__os, (int)sizeof(_Py_M__os), GET_CODE(os)}, | ||
| 123 | - {"site", _Py_M__site, (int)sizeof(_Py_M__site), GET_CODE(site)}, | ||
| 124 | - {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), GET_CODE(stat)}, | ||
| 91 | + {"_collections_abc", NULL, 0, false, GET_CODE(_collections_abc)}, | ||
| 92 | + {"_sitebuiltins", NULL, 0, false, GET_CODE(_sitebuiltins)}, | ||
| 93 | + {"genericpath", NULL, 0, false, GET_CODE(genericpath)}, | ||
| 94 | + {"ntpath", NULL, 0, false, GET_CODE(ntpath)}, | ||
| 95 | + {"posixpath", NULL, 0, false, GET_CODE(posixpath)}, | ||
| 96 | + {"os.path", NULL, 0, false, GET_CODE(posixpath)}, | ||
| 97 | + {"os", NULL, 0, false, GET_CODE(os)}, | ||
| 98 | + {"site", NULL, 0, false, GET_CODE(site)}, | ||
| 99 | + {"stat", NULL, 0, false, GET_CODE(stat)}, | ||
| 125 | 100 | ||
| 126 | 101 | /* runpy - run module with -m */ | |
| 127 | - {"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), GET_CODE(importlib_util)}, | ||
| 128 | - {"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), GET_CODE(importlib_machinery)}, | ||
| 129 | - {"runpy", _Py_M__runpy, (int)sizeof(_Py_M__runpy), GET_CODE(runpy)}, | ||
| 102 | + {"importlib.util", NULL, 0, false, GET_CODE(importlib_util)}, | ||
| 103 | + {"importlib.machinery", NULL, 0, false, GET_CODE(importlib_machinery)}, | ||
| 104 | + {"runpy", NULL, 0, false, GET_CODE(runpy)}, | ||
| 130 | 105 | {0, 0, 0} /* stdlib sentinel */ | |
| 131 | 106 | }; | |
| 132 | 107 | static const struct _frozen test_modules[] = { | |
| 133 | - {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)}, | ||
| 134 | - {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)}, | ||
| 135 | - {"__phello_alias__", _Py_M____hello__, -(int)sizeof(_Py_M____hello__), GET_CODE(__hello__)}, | ||
| 136 | - {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)}, | ||
| 137 | - {"__phello__", _Py_M____phello__, -(int)sizeof(_Py_M____phello__), GET_CODE(__phello__)}, | ||
| 138 | - {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), GET_CODE(__phello__)}, | ||
| 139 | - {"__phello__.ham", _Py_M____phello___ham, -(int)sizeof(_Py_M____phello___ham), GET_CODE(__phello___ham)}, | ||
| 140 | - {"__phello__.ham.__init__", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), GET_CODE(__phello___ham)}, | ||
| 141 | - {"__phello__.ham.eggs", _Py_M____phello___ham_eggs, (int)sizeof(_Py_M____phello___ham_eggs), GET_CODE(__phello___ham_eggs)}, | ||
| 142 | - {"__phello__.spam", _Py_M____phello___spam, (int)sizeof(_Py_M____phello___spam), GET_CODE(__phello___spam)}, | ||
| 143 | - {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only), GET_CODE(frozen_only)}, | ||
| 108 | + {"__hello__", NULL, 0, false, GET_CODE(__hello__)}, | ||
| 109 | + {"__hello_alias__", NULL, 0, false, GET_CODE(__hello__)}, | ||
| 110 | + {"__phello_alias__", NULL, 0, true, GET_CODE(__hello__)}, | ||
| 111 | + {"__phello_alias__.spam", NULL, 0, false, GET_CODE(__hello__)}, | ||
| 112 | + {"__phello__", NULL, 0, true, GET_CODE(__phello__)}, | ||
| 113 | + {"__phello__.__init__", NULL, 0, false, GET_CODE(__phello__)}, | ||
| 114 | + {"__phello__.ham", NULL, 0, true, GET_CODE(__phello___ham)}, | ||
| 115 | + {"__phello__.ham.__init__", NULL, 0, false, GET_CODE(__phello___ham)}, | ||
| 116 | + {"__phello__.ham.eggs", NULL, 0, false, GET_CODE(__phello___ham_eggs)}, | ||
| 117 | + {"__phello__.spam", NULL, 0, false, GET_CODE(__phello___spam)}, | ||
| 118 | + {"__hello_only__", NULL, 0, false, GET_CODE(frozen_only)}, | ||
| 144 | 119 | {0, 0, 0} /* test sentinel */ | |
| 145 | 120 | }; | |
| 146 | 121 | const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules; | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1297,13 +1297,21 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) | |||
| 1297 | 1297 | info->nameobj = nameobj; // borrowed | |
| 1298 | 1298 | info->data = (const char *)p->code; | |
| 1299 | 1299 | info->get_code = p->get_code; | |
| 1300 | - info->size = p->size < 0 ? -(p->size) : p->size; | ||
| 1301 | - info->is_package = p->size < 0 ? true : false; | ||
| 1300 | + info->size = p->size; | ||
| 1301 | + info->is_package = p->is_package; | ||
| 1302 | + if (p->size < 0) { | ||
| 1303 | + // backward compatibility with negative size values | ||
| 1304 | + info->size = -(p->size); | ||
| 1305 | + info->is_package = true; | ||
| 1306 | + } | ||
| 1302 | 1307 | info->origname = name; | |
| 1303 | 1308 | info->is_alias = resolve_module_alias(name, _PyImport_FrozenAliases, | |
| 1304 | 1309 | &info->origname); | |
| 1305 | 1310 | } | |
| 1306 | - | ||
| 1311 | + if (p->code == NULL && p->size == 0 && p->get_code != NULL) { | ||
| 1312 | + /* It is only deepfrozen. */ | ||
| 1313 | + return FROZEN_OKAY; | ||
| 1314 | + } | ||
| 1307 | 1315 | if (p->code == NULL) { | |
| 1308 | 1316 | /* It is frozen but marked as un-importable. */ | |
| 1309 | 1317 | return FROZEN_EXCLUDED; | |
@@ -2224,7 +2232,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name, | |||
| 2224 | 2232 | if (info.nameobj == NULL) { | |
| 2225 | 2233 | info.nameobj = name; | |
| 2226 | 2234 | } | |
| 2227 | - if (info.size == 0) { | ||
| 2235 | + if (info.size == 0 && info.get_code == NULL) { | ||
| 2228 | 2236 | /* Does not contain executable code. */ | |
| 2229 | 2237 | set_frozen_error(FROZEN_INVALID, name); | |
| 2230 | 2238 | return NULL; | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -45,19 +45,19 @@ def makefreeze(base, dict, debug=0, entry_point=None, fail_import=()): | |||
| 45 | 45 | print("freezing", mod, "...") | |
| 46 | 46 | str = marshal.dumps(m.__code__) | |
| 47 | 47 | size = len(str) | |
| 48 | + is_package = 'false' | ||
| 48 | 49 | if m.__path__: | |
| 49 | - # Indicate package by negative size | ||
| 50 | - size = -size | ||
| 51 | - done.append((mod, mangled, size)) | ||
| 50 | + is_package = 'true' | ||
| 51 | + done.append((mod, mangled, size, is_package)) | ||
| 52 | 52 | writecode(outfp, mangled, str) | |
| 53 | 53 | if debug: | |
| 54 | 54 | print("generating table of frozen modules") | |
| 55 | 55 | with bkfile.open(base + 'frozen.c', 'w') as outfp: | |
| 56 | - for mod, mangled, size in done: | ||
| 56 | + for mod, mangled, size, _ in done: | ||
| 57 | 57 | outfp.write('extern unsigned char M_%s[];\n' % mangled) | |
| 58 | 58 | outfp.write(header) | |
| 59 | - for mod, mangled, size in done: | ||
| 60 | - outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mangled, size)) | ||
| 59 | + for mod, mangled, size, is_package in done: | ||
| 60 | + outfp.write('\t{"%s", M_%s, %d, %s},\n' % (mod, mangled, size, is_package)) | ||
| 61 | 61 | outfp.write('\n') | |
| 62 | 62 | # The following modules have a NULL code pointer, indicating | |
| 63 | 63 | # that the frozen program should not search for them on the host | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -9,7 +9,7 @@ | |||
| 9 | 9 | import ntpath | |
| 10 | 10 | import posixpath | |
| 11 | 11 | import sys | |
| 12 | - | ||
| 12 | + import argparse | ||
| 13 | 13 | from update_file import updating_file_with_tmpfile | |
| 14 | 14 | ||
| 15 | 15 | ||
@@ -463,14 +463,15 @@ def replace_block(lines, start_marker, end_marker, replacements, file): | |||
| 463 | 463 | return lines[:start_pos + 1] + replacements + lines[end_pos:] | |
| 464 | 464 | ||
| 465 | 465 | ||
| 466 | - def regen_frozen(modules): | ||
| 466 | + def regen_frozen(modules, deepfreeze_only: bool): | ||
| 467 | 467 | headerlines = [] | |
| 468 | 468 | parentdir = os.path.dirname(FROZEN_FILE) | |
| 469 | - for src in _iter_sources(modules): | ||
| 470 | - # Adding a comment to separate sections here doesn't add much, | ||
| 471 | - # so we don't. | ||
| 472 | - header = relpath_for_posix_display(src.frozenfile, parentdir) | ||
| 473 | - headerlines.append(f'#include "{header}"') | ||
| 469 | + if not deepfreeze_only: | ||
| 470 | + for src in _iter_sources(modules): | ||
| 471 | + # Adding a comment to separate sections here doesn't add much, | ||
| 472 | + # so we don't. | ||
| 473 | + header = relpath_for_posix_display(src.frozenfile, parentdir) | ||
| 474 | + headerlines.append(f'#include "{header}"') | ||
| 474 | 475 | ||
| 475 | 476 | externlines = [] | |
| 476 | 477 | bootstraplines = [] | |
@@ -500,9 +501,13 @@ def regen_frozen(modules): | |||
| 500 | 501 | externlines.append("extern PyObject *%s(void);" % get_code_name) | |
| 501 | 502 | ||
| 502 | 503 | symbol = mod.symbol | |
| 503 | - pkg = '-' if mod.ispkg else '' | ||
| 504 | - line = ('{"%s", %s, %s(int)sizeof(%s), GET_CODE(%s)},' | ||
| 505 | - ) % (mod.name, symbol, pkg, symbol, code_name) | ||
| 504 | + pkg = 'true' if mod.ispkg else 'false' | ||
| 505 | + if deepfreeze_only: | ||
| 506 | + line = ('{"%s", NULL, 0, %s, GET_CODE(%s)},' | ||
| 507 | + ) % (mod.name, pkg, code_name) | ||
| 508 | + else: | ||
| 509 | + line = ('{"%s", %s, (int)sizeof(%s), %s, GET_CODE(%s)},' | ||
| 510 | + ) % (mod.name, symbol, symbol, pkg, code_name) | ||
| 506 | 511 | lines.append(line) | |
| 507 | 512 | ||
| 508 | 513 | if mod.isalias: | |
@@ -710,18 +715,19 @@ def regen_pcbuild(modules): | |||
| 710 | 715 | ####################################### | |
| 711 | 716 | # the script | |
| 712 | 717 | ||
| 713 | - def main(): | ||
| 718 | + def main(deepfreeze_only: bool): | ||
| 714 | 719 | # Expand the raw specs, preserving order. | |
| 715 | 720 | modules = list(parse_frozen_specs()) | |
| 716 | 721 | ||
| 717 | 722 | # Regen build-related files. | |
| 718 | 723 | regen_makefile(modules) | |
| 719 | 724 | regen_pcbuild(modules) | |
| 720 | - regen_frozen(modules) | ||
| 725 | + regen_frozen(modules, deepfreeze_only) | ||
| 721 | 726 | ||
| 722 | 727 | ||
| 723 | 728 | if __name__ == '__main__': | |
| 724 | - argv = sys.argv[1:] | ||
| 725 | - if argv: | ||
| 726 | - sys.exit(f'ERROR: got unexpected args {argv}') | ||
| 727 | - main() | ||
| 729 | + parser = argparse.ArgumentParser() | ||
| 730 | + parser.add_argument("--deepfreeze-only", action="store_true", | ||
| 731 | + help="Only use deepfrozen modules", default=True) | ||
| 732 | + args = parser.parse_args() | ||
| 733 | + main(args.deepfreeze_only) | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments