| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent a5785b1 commit c39211f
5 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -99,6 +99,13 @@ PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co, | |||
| 99 | 99 | PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, | |
| 100 | 100 | PyObject *names, PyObject *lineno_obj); | |
| 101 | 101 | ||
| 102 | + /* List of weak references to all code objects. The list is used by | ||
| 103 | + initfsencoding() to redecode code filenames at startup if the filesystem | ||
| 104 | + encoding changes. At initfsencoding() exit, the list is set to NULL and it | ||
| 105 | + is no more used. */ | ||
| 106 | + | ||
| 107 | + extern PyObject *_Py_code_object_list; | ||
| 108 | + | ||
| 102 | 109 | #ifdef __cplusplus | |
| 103 | 110 | } | |
| 104 | 111 | #endif | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -5,6 +5,8 @@ | |||
| 5 | 5 | #define NAME_CHARS \ | |
| 6 | 6 | "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" | |
| 7 | 7 | ||
| 8 | + PyObject *_Py_code_object_list = NULL; | ||
| 9 | + | ||
| 8 | 10 | /* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */ | |
| 9 | 11 | ||
| 10 | 12 | static int | |
@@ -109,8 +111,23 @@ PyCode_New(int argcount, int kwonlyargcount, | |||
| 109 | 111 | co->co_lnotab = lnotab; | |
| 110 | 112 | co->co_zombieframe = NULL; | |
| 111 | 113 | co->co_weakreflist = NULL; | |
| 114 | + | ||
| 115 | + if (_Py_code_object_list != NULL) { | ||
| 116 | + int err; | ||
| 117 | + PyObject *ref = PyWeakref_NewRef((PyObject*)co, NULL); | ||
| 118 | + if (ref == NULL) | ||
| 119 | + goto error; | ||
| 120 | + err = PyList_Append(_Py_code_object_list, ref); | ||
| 121 | + Py_DECREF(ref); | ||
| 122 | + if (err) | ||
| 123 | + goto error; | ||
| 124 | + } | ||
| 112 | 125 | } | |
| 113 | 126 | return co; | |
| 127 | + | ||
| 128 | + error: | ||
| 129 | + Py_DECREF(co); | ||
| 130 | + return NULL; | ||
| 114 | 131 | } | |
| 115 | 132 | ||
| 116 | 133 | PyCodeObject * | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1604,6 +1604,10 @@ _Py_ReadyTypes(void) | |||
| 1604 | 1604 | if (PyType_Ready(&PyCode_Type) < 0) | |
| 1605 | 1605 | Py_FatalError("Can't initialize code type"); | |
| 1606 | 1606 | ||
| 1607 | + _Py_code_object_list = PyList_New(0); | ||
| 1608 | + if (_Py_code_object_list == NULL) | ||
| 1609 | + Py_FatalError("Can't initialize code type"); | ||
| 1610 | + | ||
| 1607 | 1611 | if (PyType_Ready(&PyFrame_Type) < 0) | |
| 1608 | 1612 | Py_FatalError("Can't initialize frame type"); | |
| 1609 | 1613 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1510,10 +1510,14 @@ PyObject *PyUnicode_EncodeFSDefault(PyObject *unicode) | |||
| 1510 | 1510 | return PyUnicode_AsEncodedString(unicode, | |
| 1511 | 1511 | Py_FileSystemDefaultEncoding, | |
| 1512 | 1512 | "surrogateescape"); | |
| 1513 | - } else | ||
| 1513 | + } | ||
| 1514 | + else { | ||
| 1515 | + /* if you change the default encoding, update also | ||
| 1516 | + PyUnicode_DecodeFSDefaultAndSize() and redecode_filenames() */ | ||
| 1514 | 1517 | return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), | |
| 1515 | 1518 | PyUnicode_GET_SIZE(unicode), | |
| 1516 | 1519 | "surrogateescape"); | |
| 1520 | + } | ||
| 1517 | 1521 | } | |
| 1518 | 1522 | ||
| 1519 | 1523 | PyObject *PyUnicode_AsEncodedString(PyObject *unicode, | |
@@ -1680,6 +1684,8 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) | |||
| 1680 | 1684 | "surrogateescape"); | |
| 1681 | 1685 | } | |
| 1682 | 1686 | else { | |
| 1687 | + /* if you change the default encoding, update also | ||
| 1688 | + PyUnicode_EncodeFSDefault() and redecode_filenames() */ | ||
| 1683 | 1689 | return PyUnicode_DecodeUTF8(s, size, "surrogateescape"); | |
| 1684 | 1690 | } | |
| 1685 | 1691 | } | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -719,6 +719,259 @@ initmain(void) | |||
| 719 | 719 | } | |
| 720 | 720 | } | |
| 721 | 721 | ||
| 722 | + /* Redecode a filename from the default filesystem encoding (utf-8) to | ||
| 723 | + 'new_encoding' encoding with 'errors' error handler */ | ||
| 724 | + static PyObject* | ||
| 725 | + redecode_filename(PyObject *file, const char *new_encoding, | ||
| 726 | + const char *errors) | ||
| 727 | + { | ||
| 728 | + PyObject *file_bytes = NULL, *new_file = NULL; | ||
| 729 | + | ||
| 730 | + file_bytes = PyUnicode_EncodeFSDefault(file); | ||
| 731 | + if (file_bytes == NULL) | ||
| 732 | + return NULL; | ||
| 733 | + new_file = PyUnicode_Decode( | ||
| 734 | + PyBytes_AsString(file_bytes), | ||
| 735 | + PyBytes_GET_SIZE(file_bytes), | ||
| 736 | + new_encoding, | ||
| 737 | + errors); | ||
| 738 | + Py_DECREF(file_bytes); | ||
| 739 | + return new_file; | ||
| 740 | + } | ||
| 741 | + | ||
| 742 | + /* Redecode a path list */ | ||
| 743 | + static int | ||
| 744 | + redecode_path_list(PyObject *paths, | ||
| 745 | + const char *new_encoding, const char *errors) | ||
| 746 | + { | ||
| 747 | + PyObject *filename, *new_filename; | ||
| 748 | + Py_ssize_t i, size; | ||
| 749 | + | ||
| 750 | + size = PyList_Size(paths); | ||
| 751 | + for (i=0; i < size; i++) { | ||
| 752 | + filename = PyList_GetItem(paths, i); | ||
| 753 | + if (filename == NULL) | ||
| 754 | + return -1; | ||
| 755 | + | ||
| 756 | + new_filename = redecode_filename(filename, new_encoding, errors); | ||
| 757 | + if (new_filename == NULL) | ||
| 758 | + return -1; | ||
| 759 | + if (PyList_SetItem(paths, i, new_filename)) { | ||
| 760 | + Py_DECREF(new_filename); | ||
| 761 | + return -1; | ||
| 762 | + } | ||
| 763 | + } | ||
| 764 | + return 0; | ||
| 765 | + } | ||
| 766 | + | ||
| 767 | + /* Redecode __file__ and __path__ attributes of sys.modules */ | ||
| 768 | + static int | ||
| 769 | + redecode_sys_modules(const char *new_encoding, const char *errors) | ||
| 770 | + { | ||
| 771 | + PyInterpreterState *interp; | ||
| 772 | + PyObject *modules, *values, *file, *new_file, *paths; | ||
| 773 | + PyObject *iter = NULL, *module = NULL; | ||
| 774 | + | ||
| 775 | + interp = PyThreadState_GET()->interp; | ||
| 776 | + modules = interp->modules; | ||
| 777 | + | ||
| 778 | + values = PyObject_CallMethod(modules, "values", ""); | ||
| 779 | + if (values == NULL) | ||
| 780 | + goto error; | ||
| 781 | + | ||
| 782 | + iter = PyObject_GetIter(values); | ||
| 783 | + Py_DECREF(values); | ||
| 784 | + if (iter == NULL) | ||
| 785 | + goto error; | ||
| 786 | + | ||
| 787 | + while (1) | ||
| 788 | + { | ||
| 789 | + module = PyIter_Next(iter); | ||
| 790 | + if (module == NULL) { | ||
| 791 | + if (PyErr_Occurred()) | ||
| 792 | + goto error; | ||
| 793 | + else | ||
| 794 | + break; | ||
| 795 | + } | ||
| 796 | + | ||
| 797 | + file = PyModule_GetFilenameObject(module); | ||
| 798 | + if (file != NULL) { | ||
| 799 | + new_file = redecode_filename(file, new_encoding, errors); | ||
| 800 | + Py_DECREF(file); | ||
| 801 | + if (new_file == NULL) | ||
| 802 | + goto error; | ||
| 803 | + if (PyObject_SetAttrString(module, "__file__", new_file)) { | ||
| 804 | + Py_DECREF(new_file); | ||
| 805 | + goto error; | ||
| 806 | + } | ||
| 807 | + Py_DECREF(new_file); | ||
| 808 | + } | ||
| 809 | + else | ||
| 810 | + PyErr_Clear(); | ||
| 811 | + | ||
| 812 | + paths = PyObject_GetAttrString(module, "__path__"); | ||
| 813 | + if (paths != NULL) { | ||
| 814 | + if (redecode_path_list(paths, new_encoding, errors)) | ||
| 815 | + goto error; | ||
| 816 | + } | ||
| 817 | + else | ||
| 818 | + PyErr_Clear(); | ||
| 819 | + | ||
| 820 | + Py_CLEAR(module); | ||
| 821 | + } | ||
| 822 | + Py_CLEAR(iter); | ||
| 823 | + return 0; | ||
| 824 | + | ||
| 825 | + error: | ||
| 826 | + Py_XDECREF(iter); | ||
| 827 | + Py_XDECREF(module); | ||
| 828 | + return -1; | ||
| 829 | + } | ||
| 830 | + | ||
| 831 | + /* Redecode sys.path_importer_cache keys */ | ||
| 832 | + static int | ||
| 833 | + redecode_sys_path_importer_cache(const char *new_encoding, const char *errors) | ||
| 834 | + { | ||
| 835 | + PyObject *path_importer_cache, *items, *item, *path, *importer, *new_path; | ||
| 836 | + PyObject *new_cache = NULL, *iter = NULL; | ||
| 837 | + | ||
| 838 | + path_importer_cache = PySys_GetObject("path_importer_cache"); | ||
| 839 | + if (path_importer_cache == NULL) | ||
| 840 | + goto error; | ||
| 841 | + | ||
| 842 | + items = PyObject_CallMethod(path_importer_cache, "items", ""); | ||
| 843 | + if (items == NULL) | ||
| 844 | + goto error; | ||
| 845 | + | ||
| 846 | + iter = PyObject_GetIter(items); | ||
| 847 | + Py_DECREF(items); | ||
| 848 | + if (iter == NULL) | ||
| 849 | + goto error; | ||
| 850 | + | ||
| 851 | + new_cache = PyDict_New(); | ||
| 852 | + if (new_cache == NULL) | ||
| 853 | + goto error; | ||
| 854 | + | ||
| 855 | + while (1) | ||
| 856 | + { | ||
| 857 | + item = PyIter_Next(iter); | ||
| 858 | + if (item == NULL) { | ||
| 859 | + if (PyErr_Occurred()) | ||
| 860 | + goto error; | ||
| 861 | + else | ||
| 862 | + break; | ||
| 863 | + } | ||
| 864 | + path = PyTuple_GET_ITEM(item, 0); | ||
| 865 | + importer = PyTuple_GET_ITEM(item, 1); | ||
| 866 | + | ||
| 867 | + new_path = redecode_filename(path, new_encoding, errors); | ||
| 868 | + if (new_path == NULL) | ||
| 869 | + goto error; | ||
| 870 | + if (PyDict_SetItem(new_cache, new_path, importer)) { | ||
| 871 | + Py_DECREF(new_path); | ||
| 872 | + goto error; | ||
| 873 | + } | ||
| 874 | + Py_DECREF(new_path); | ||
| 875 | + } | ||
| 876 | + Py_CLEAR(iter); | ||
| 877 | + if (PySys_SetObject("path_importer_cache", new_cache)) | ||
| 878 | + goto error; | ||
| 879 | + Py_CLEAR(new_cache); | ||
| 880 | + return 0; | ||
| 881 | + | ||
| 882 | + error: | ||
| 883 | + Py_XDECREF(iter); | ||
| 884 | + Py_XDECREF(new_cache); | ||
| 885 | + return -1; | ||
| 886 | + } | ||
| 887 | + | ||
| 888 | + /* Redecode co_filename attribute of all code objects */ | ||
| 889 | + static int | ||
| 890 | + redecode_code_objects(const char *new_encoding, const char *errors) | ||
| 891 | + { | ||
| 892 | + Py_ssize_t i, len; | ||
| 893 | + PyCodeObject *co; | ||
| 894 | + PyObject *ref, *new_file; | ||
| 895 | + | ||
| 896 | + len = Py_SIZE(_Py_code_object_list); | ||
| 897 | + for (i=0; i < len; i++) { | ||
| 898 | + ref = PyList_GET_ITEM(_Py_code_object_list, i); | ||
| 899 | + co = (PyCodeObject *)PyWeakref_GetObject(ref); | ||
| 900 | + if ((PyObject*)co == Py_None) | ||
| 901 | + continue; | ||
| 902 | + if (co == NULL) | ||
| 903 | + return -1; | ||
| 904 | + | ||
| 905 | + new_file = redecode_filename(co->co_filename, new_encoding, errors); | ||
| 906 | + if (new_file == NULL) | ||
| 907 | + return -1; | ||
| 908 | + Py_DECREF(co->co_filename); | ||
| 909 | + co->co_filename = new_file; | ||
| 910 | + } | ||
| 911 | + Py_CLEAR(_Py_code_object_list); | ||
| 912 | + return 0; | ||
| 913 | + } | ||
| 914 | + | ||
| 915 | + /* Redecode the filenames of all modules (__file__ and __path__ attributes), | ||
| 916 | + all code objects (co_filename attribute), sys.path, sys.meta_path, | ||
| 917 | + sys.executable and sys.path_importer_cache (keys) when the filesystem | ||
| 918 | + encoding changes from the default encoding (utf-8) to new_encoding */ | ||
| 919 | + static int | ||
| 920 | + redecode_filenames(const char *new_encoding) | ||
| 921 | + { | ||
| 922 | + char *errors; | ||
| 923 | + PyObject *paths, *executable, *new_executable; | ||
| 924 | + | ||
| 925 | + /* PyUnicode_DecodeFSDefault() and PyUnicode_EncodeFSDefault() do already | ||
| 926 | + use utf-8 if Py_FileSystemDefaultEncoding is NULL */ | ||
| 927 | + if (strcmp(new_encoding, "utf-8") == 0) | ||
| 928 | + return 0; | ||
| 929 | + | ||
| 930 | + if (strcmp(new_encoding, "mbcs") != 0) | ||
| 931 | + errors = "surrogateescape"; | ||
| 932 | + else | ||
| 933 | + errors = NULL; | ||
| 934 | + | ||
| 935 | + /* sys.modules */ | ||
| 936 | + if (redecode_sys_modules(new_encoding, errors)) | ||
| 937 | + return -1; | ||
| 938 | + | ||
| 939 | + /* sys.path and sys.meta_path */ | ||
| 940 | + paths = PySys_GetObject("path"); | ||
| 941 | + if (paths != NULL) { | ||
| 942 | + if (redecode_path_list(paths, new_encoding, errors)) | ||
| 943 | + return -1; | ||
| 944 | + } | ||
| 945 | + paths = PySys_GetObject("meta_path"); | ||
| 946 | + if (paths != NULL) { | ||
| 947 | + if (redecode_path_list(paths, new_encoding, errors)) | ||
| 948 | + return -1; | ||
| 949 | + } | ||
| 950 | + | ||
| 951 | + /* sys.executable */ | ||
| 952 | + executable = PySys_GetObject("executable"); | ||
| 953 | + if (executable == NULL) | ||
| 954 | + return -1; | ||
| 955 | + new_executable = redecode_filename(executable, new_encoding, errors); | ||
| 956 | + if (new_executable == NULL) | ||
| 957 | + return -1; | ||
| 958 | + if (PySys_SetObject("executable", new_executable)) { | ||
| 959 | + Py_DECREF(new_executable); | ||
| 960 | + return -1; | ||
| 961 | + } | ||
| 962 | + Py_DECREF(new_executable); | ||
| 963 | + | ||
| 964 | + /* sys.path_importer_cache */ | ||
| 965 | + if (redecode_sys_path_importer_cache(new_encoding, errors)) | ||
| 966 | + return -1; | ||
| 967 | + | ||
| 968 | + /* code objects */ | ||
| 969 | + if (redecode_code_objects(new_encoding, errors)) | ||
| 970 | + return -1; | ||
| 971 | + | ||
| 972 | + return 0; | ||
| 973 | + } | ||
| 974 | + | ||
| 722 | 975 | static void | |
| 723 | 976 | initfsencoding(void) | |
| 724 | 977 | { | |
@@ -744,8 +997,11 @@ initfsencoding(void) | |||
| 744 | 997 | codeset = get_codeset(); | |
| 745 | 998 | } | |
| 746 | 999 | if (codeset != NULL) { | |
| 1000 | + if (redecode_filenames(codeset)) | ||
| 1001 | + Py_FatalError("Py_Initialize: can't redecode filenames"); | ||
| 747 | 1002 | Py_FileSystemDefaultEncoding = codeset; | |
| 748 | 1003 | Py_HasFileSystemDefaultEncoding = 0; | |
| 1004 | + Py_CLEAR(_Py_code_object_list); | ||
| 749 | 1005 | return; | |
| 750 | 1006 | } else { | |
| 751 | 1007 | fprintf(stderr, "Unable to get the locale encoding:\n"); | |
@@ -758,6 +1014,8 @@ initfsencoding(void) | |||
| 758 | 1014 | } | |
| 759 | 1015 | #endif | |
| 760 | 1016 | ||
| 1017 | + Py_CLEAR(_Py_code_object_list); | ||
| 1018 | + | ||
| 761 | 1019 | /* the encoding is mbcs, utf-8 or ascii */ | |
| 762 | 1020 | codec = _PyCodec_Lookup(Py_FileSystemDefaultEncoding); | |
| 763 | 1021 | if (!codec) { | |
| Back | FazBrowse Home | New Git URL |
0 commit comments