| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -76,11 +76,12 @@ def arbitrary_address(family): | |||
| 76 | 76 | if family == 'AF_INET': | |
| 77 | 77 | return ('localhost', 0) | |
| 78 | 78 | elif family == 'AF_UNIX': | |
| 79 | - # NOTE: util.get_temp_dir() is a 0o700 per-process directory. A | ||
| 80 | - # mktemp-style ToC vs ToU concern is not important; bind() surfaces | ||
| 81 | - # the extremely unlikely collision as EADDRINUSE. | ||
| 79 | + # NOTE: util.get_temp_dir() is a 0o700 per-process directory. | ||
| 80 | + # A mktemp-style ToC vs ToU concern is not important as bind() | ||
| 81 | + # surfaces the extremely unlikely collision as EADDRINUSE. | ||
| 82 | + suffix = os.urandom(util._TMPSOCK_SUFFIXLEN // 2).hex() | ||
| 82 | 83 | return os.path.join(util.get_temp_dir(), | |
| 83 | - f'sock-{os.urandom(6).hex()}') | ||
| 84 | + f"{util._TMPSOCK_PREFIX}{suffix}") | ||
| 84 | 85 | elif family == 'AF_PIPE': | |
| 85 | 86 | return (r'\\.\pipe\pyc-%d-%d-%s' % | |
| 86 | 87 | (os.getpid(), next(_mmap_counter), os.urandom(8).hex())) | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -143,6 +143,41 @@ def is_abstract_socket_namespace(address): | |||
| 143 | 143 | # On Windows platforms, we do not create AF_UNIX sockets. | |
| 144 | 144 | _SUN_PATH_MAX = None if os.name == 'nt' else 92 | |
| 145 | 145 | ||
| 146 | + # The temporary socket file path (without NULL terminator) is | ||
| 147 | + # | ||
| 148 | + # TEMPDIR/{_TMPPYMP_PREFIX}{S1}/{_TMPSOCK_PREFIX}{S2} | ||
| 149 | + # | ||
| 150 | + # where | ||
| 151 | + # | ||
| 152 | + # - S1 is a random suffix of length 8 generated by get_temp_dir(). | ||
| 153 | + # This is used to generate the temporary sub-directory that will | ||
| 154 | + # contains the socket file. Update _TMPPYMP_SUFFIXLEN accordingly | ||
| 155 | + # if the random suffix length chosen by tempfile.mkdtemp() changes. | ||
| 156 | + # | ||
| 157 | + # - S2 is a random suffix of length 2N generated by os.urandom(N).hex() | ||
| 158 | + # in multiprocessing.connection.arbitrary_address(). The value of N | ||
| 159 | + # is not fixed and may vary across Python versions as to minimize | ||
| 160 | + # collisions and path lengths. | ||
| 161 | + # | ||
| 162 | + _TMPPYMP_PREFIX = "pymp-" | ||
| 163 | + _TMPPYMP_PREFIXLEN = len(_TMPPYMP_PREFIX) | ||
| 164 | + _TMPPYMP_SUFFIXLEN = 8 | ||
| 165 | + _TMPSOCK_PREFIX = "sock-" | ||
| 166 | + _TMPSOCK_PREFIXLEN = len(_TMPSOCK_PREFIX) | ||
| 167 | + _TMPSOCK_SUFFIXLEN = 12 | ||
| 168 | + | ||
| 169 | + # Length of the socket filepath from the chosen temporary directory, | ||
| 170 | + # including a leading path separator and the path separator between | ||
| 171 | + # the temporary subdirectory and the socket filename. | ||
| 172 | + _SUN_PATH_LEN_RESERVED = ( | ||
| 173 | + len(os.path.sep) + _TMPPYMP_PREFIXLEN + _TMPPYMP_SUFFIXLEN + | ||
| 174 | + len(os.path.sep) + _TMPSOCK_PREFIXLEN + _TMPSOCK_SUFFIXLEN | ||
| 175 | + ) | ||
| 176 | + assert ( | ||
| 177 | + _SUN_PATH_MAX is None | ||
| 178 | + or _SUN_PATH_LEN_RESERVED < _SUN_PATH_MAX | ||
| 179 | + ) | ||
| 180 | + | ||
| 146 | 181 | ||
| 147 | 182 | def _remove_temp_dir(rmtree, tempdir): | |
| 148 | 183 | rmtree(tempdir) | |
@@ -161,24 +196,16 @@ def _get_base_temp_dir(tempfile): | |||
| 161 | 196 | """ | |
| 162 | 197 | if os.name == 'nt': | |
| 163 | 198 | return None | |
| 164 | - # Most of the time, the default temporary directory is /tmp. Thus, | ||
| 165 | - # listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do | ||
| 166 | - # not have a path length exceeding SUN_PATH_MAX. | ||
| 199 | + # Most of the time, the default temporary directory is /tmp. As such, | ||
| 200 | + # the path length of listener sockets files does not exceed SUN_PATH_MAX. | ||
| 167 | 201 | # | |
| 168 | 202 | # If users specify their own temporary directory, we may be unable | |
| 169 | 203 | # to create those files. Therefore, we fall back to the system-wide | |
| 170 | 204 | # temporary directory /tmp, assumed to exist on POSIX systems. | |
| 171 | 205 | # | |
| 172 | 206 | # See https://github.com/python/cpython/issues/132124. | |
| 173 | 207 | base_tempdir = tempfile.gettempdir() | |
| 174 | - # Files created in a temporary directory are suffixed by a string | ||
| 175 | - # generated by tempfile._RandomNameSequence, which, by design, | ||
| 176 | - # is 8 characters long. | ||
| 177 | - # | ||
| 178 | - # Thus, the socket file path length (without NULL terminator) will be: | ||
| 179 | - # | ||
| 180 | - # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') | ||
| 181 | - sun_path_len = len(base_tempdir) + 14 + 14 | ||
| 208 | + sun_path_len = len(base_tempdir) + _SUN_PATH_LEN_RESERVED | ||
| 182 | 209 | # Strict inequality to account for the NULL terminator. | |
| 183 | 210 | # See https://github.com/python/cpython/issues/140734. | |
| 184 | 211 | if sun_path_len < _SUN_PATH_MAX: | |
@@ -206,8 +233,8 @@ def _get_base_temp_dir(tempfile): | |||
| 206 | 233 | # not be able to write socket files out there. | |
| 207 | 234 | return base_tempdir | |
| 208 | 235 | warn("Ignoring user-defined temporary directory: %s", base_tempdir) | |
| 209 | - # at most max(map(len, dirlist)) + 14 + 14 = 36 characters | ||
| 210 | - assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX | ||
| 236 | + # The following assertion must be satisfied with the chosen constants. | ||
| 237 | + assert len(base_system_tempdir) + _SUN_PATH_LEN_RESERVED < _SUN_PATH_MAX | ||
| 211 | 238 | return base_system_tempdir | |
| 212 | 239 | ||
| 213 | 240 | ||
@@ -218,7 +245,7 @@ def get_temp_dir(): | |||
| 218 | 245 | # Deferred until first use to keep importing this module cheap. | |
| 219 | 246 | import shutil, tempfile | |
| 220 | 247 | base_tempdir = _get_base_temp_dir(tempfile) | |
| 221 | - tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) | ||
| 248 | + tempdir = tempfile.mkdtemp(prefix=_TMPPYMP_PREFIX, dir=base_tempdir) | ||
| 222 | 249 | info('created temp directory %s', tempdir) | |
| 223 | 250 | # keep a strong reference to shutil.rmtree(), since the finalizer | |
| 224 | 251 | # can be called late during Python shutdown | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,7 +1,47 @@ | |||
| 1 | + import os | ||
| 2 | + import re | ||
| 1 | 3 | import unittest | |
| 4 | + from multiprocessing.util import _SUN_PATH_MAX | ||
| 2 | 5 | from test._test_multiprocessing import install_tests_in_module_dict | |
| 6 | + from test.support import os_helper, script_helper | ||
| 3 | 7 | ||
| 4 | 8 | install_tests_in_module_dict(globals(), 'forkserver', exclude_types=True) | |
| 5 | 9 | ||
| 10 | + | ||
| 11 | + class TestForkServerConfiguration(unittest.TestCase): | ||
| 12 | + def test_respect_sun_path_max(self): | ||
| 13 | + # Ensure that the calculation for temporary filepath lengths is correct. | ||
| 14 | + # See https://github.com/python/cpython/issues/149527. | ||
| 15 | + | ||
| 16 | + cmd = '''if 1: | ||
| 17 | + from multiprocessing.connection import arbitrary_address | ||
| 18 | + from multiprocessing.util import get_temp_dir | ||
| 19 | + if __name__ == "__main__": | ||
| 20 | + print(get_temp_dir()) | ||
| 21 | + print(arbitrary_address("AF_UNIX")) | ||
| 22 | + ''' | ||
| 23 | + with os_helper.temp_dir() as root: | ||
| 24 | + self.assertLess(len(root), _SUN_PATH_MAX) | ||
| 25 | + _, out, _ = script_helper.assert_python_ok('-c', cmd, TMPDIR=root) | ||
| 26 | + res = out.decode().strip().splitlines() | ||
| 27 | + self.assertEqual(len(res), 2) | ||
| 28 | + | ||
| 29 | + # get_temp_dir() with a TMPDIR that has a too large length ends up | ||
| 30 | + # NOT being used as it is not possible to use it. In particular, | ||
| 31 | + # the corresponding path will fallback to one of the "built-in" | ||
| 32 | + # ones such as '/tmp', '/var/tmp', or '/usr/tmp'. | ||
| 33 | + root_regex = "|".join(map(re.escape, [root, "/tmp", "/var/tmp", "/usr/tmp"])) | ||
| 34 | + | ||
| 35 | + temp_pymp = res[0] | ||
| 36 | + self.assertLess(len(temp_pymp), _SUN_PATH_MAX) | ||
| 37 | + temp_pymp_regex = os.path.join(root_regex, r"pymp-\w{8}") | ||
| 38 | + self.assertRegex(temp_pymp, temp_pymp_regex) | ||
| 39 | + | ||
| 40 | + temp_sock = res[1] | ||
| 41 | + self.assertLess(len(temp_sock), _SUN_PATH_MAX) | ||
| 42 | + temp_sock_regex = os.path.join(temp_pymp_regex, r"sock-[0-9a-fA-F]{12}") | ||
| 43 | + self.assertRegex(temp_sock, temp_sock_regex) | ||
| 44 | + | ||
| 45 | + | ||
| 6 | 46 | if __name__ == '__main__': | |
| 7 | 47 | unittest.main() | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,3 @@ | |||
| 1 | + :mod:`multiprocessing`: fix creation of temporary socket files following | ||
| 2 | + :gh:`137335` (`PR#148578 <https://github.com/python/cpython/pull/148578>`__ | ||
| 3 | + and corresponding backports). Patch by Bénédikt Tran. | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments