FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

[3.14] gh-149527: fix creation of temporary socket files post GH-148578 (GH-149597) by miss-islington · Pull Request #160270 · python/cpython · GitHub

Repository navigation

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .py  (3) .rst  (1) All 2 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
9 changes: 5 additions & 4 deletions Lib/multiprocessing/connection.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,12 @@ def arbitrary_address(family):
if family == 'AF_INET':
return ('localhost', 0)
elif family == 'AF_UNIX':
# NOTE: util.get_temp_dir() is a 0o700 per-process directory. A
# mktemp-style ToC vs ToU concern is not important; bind() surfaces
# the extremely unlikely collision as EADDRINUSE.
# NOTE: util.get_temp_dir() is a 0o700 per-process directory.
# A mktemp-style ToC vs ToU concern is not important as bind()
# surfaces the extremely unlikely collision as EADDRINUSE.
suffix = os.urandom(util._TMPSOCK_SUFFIXLEN // 2).hex()
return os.path.join(util.get_temp_dir(),
f'sock-{os.urandom(6).hex()}')
f"{util._TMPSOCK_PREFIX}{suffix}")
elif family == 'AF_PIPE':
return (r'\\.\pipe\pyc-%d-%d-%s' %
(os.getpid(), next(_mmap_counter), os.urandom(8).hex()))
Expand Down
55 changes: 41 additions & 14 deletions Lib/multiprocessing/util.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,41 @@ def is_abstract_socket_namespace(address):
# On Windows platforms, we do not create AF_UNIX sockets.
_SUN_PATH_MAX = None if os.name == 'nt' else 92

# The temporary socket file path (without NULL terminator) is
#
# TEMPDIR/{_TMPPYMP_PREFIX}{S1}/{_TMPSOCK_PREFIX}{S2}
#
# where
#
# - S1 is a random suffix of length 8 generated by get_temp_dir().
# This is used to generate the temporary sub-directory that will
# contains the socket file. Update _TMPPYMP_SUFFIXLEN accordingly
# if the random suffix length chosen by tempfile.mkdtemp() changes.
#
# - S2 is a random suffix of length 2N generated by os.urandom(N).hex()
# in multiprocessing.connection.arbitrary_address(). The value of N
# is not fixed and may vary across Python versions as to minimize
# collisions and path lengths.
#
_TMPPYMP_PREFIX = "pymp-"
_TMPPYMP_PREFIXLEN = len(_TMPPYMP_PREFIX)
_TMPPYMP_SUFFIXLEN = 8
_TMPSOCK_PREFIX = "sock-"
_TMPSOCK_PREFIXLEN = len(_TMPSOCK_PREFIX)
_TMPSOCK_SUFFIXLEN = 12

# Length of the socket filepath from the chosen temporary directory,
# including a leading path separator and the path separator between
# the temporary subdirectory and the socket filename.
_SUN_PATH_LEN_RESERVED = (
len(os.path.sep) + _TMPPYMP_PREFIXLEN + _TMPPYMP_SUFFIXLEN +
len(os.path.sep) + _TMPSOCK_PREFIXLEN + _TMPSOCK_SUFFIXLEN
)
assert (
_SUN_PATH_MAX is None
or _SUN_PATH_LEN_RESERVED < _SUN_PATH_MAX
)


def _remove_temp_dir(rmtree, tempdir):
rmtree(tempdir)
Expand All @@ -161,24 +196,16 @@ def _get_base_temp_dir(tempfile):
"""
if os.name == 'nt':
return None
# Most of the time, the default temporary directory is /tmp. Thus,
# listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do
# not have a path length exceeding SUN_PATH_MAX.
# Most of the time, the default temporary directory is /tmp. As such,
# the path length of listener sockets files does not exceed SUN_PATH_MAX.
#
# If users specify their own temporary directory, we may be unable
# to create those files. Therefore, we fall back to the system-wide
# temporary directory /tmp, assumed to exist on POSIX systems.
#
# See https://github.com/python/cpython/issues/132124.
base_tempdir = tempfile.gettempdir()
# Files created in a temporary directory are suffixed by a string
# generated by tempfile._RandomNameSequence, which, by design,
# is 8 characters long.
#
# Thus, the socket file path length (without NULL terminator) will be:
#
# len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX')
sun_path_len = len(base_tempdir) + 14 + 14
sun_path_len = len(base_tempdir) + _SUN_PATH_LEN_RESERVED
# Strict inequality to account for the NULL terminator.
# See https://github.com/python/cpython/issues/140734.
if sun_path_len < _SUN_PATH_MAX:
Expand Down Expand Up @@ -206,8 +233,8 @@ def _get_base_temp_dir(tempfile):
# not be able to write socket files out there.
return base_tempdir
warn("Ignoring user-defined temporary directory: %s", base_tempdir)
# at most max(map(len, dirlist)) + 14 + 14 = 36 characters
assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX
# The following assertion must be satisfied with the chosen constants.
assert len(base_system_tempdir) + _SUN_PATH_LEN_RESERVED < _SUN_PATH_MAX
return base_system_tempdir


Expand All @@ -218,7 +245,7 @@ def get_temp_dir():
# Deferred until first use to keep importing this module cheap.
import shutil, tempfile
base_tempdir = _get_base_temp_dir(tempfile)
tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir)
tempdir = tempfile.mkdtemp(prefix=_TMPPYMP_PREFIX, dir=base_tempdir)
info('created temp directory %s', tempdir)
# keep a strong reference to shutil.rmtree(), since the finalizer
# can be called late during Python shutdown
Expand Down
40 changes: 40 additions & 0 deletions Lib/test/test_multiprocessing_forkserver/test_misc.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -1,7 +1,47 @@
import os
import re
import unittest
from multiprocessing.util import _SUN_PATH_MAX
from test._test_multiprocessing import install_tests_in_module_dict
from test.support import os_helper, script_helper

install_tests_in_module_dict(globals(), 'forkserver', exclude_types=True)


class TestForkServerConfiguration(unittest.TestCase):
def test_respect_sun_path_max(self):
# Ensure that the calculation for temporary filepath lengths is correct.
# See https://github.com/python/cpython/issues/149527.

cmd = '''if 1:
from multiprocessing.connection import arbitrary_address
from multiprocessing.util import get_temp_dir
if __name__ == "__main__":
print(get_temp_dir())
print(arbitrary_address("AF_UNIX"))
'''
with os_helper.temp_dir() as root:
self.assertLess(len(root), _SUN_PATH_MAX)
_, out, _ = script_helper.assert_python_ok('-c', cmd, TMPDIR=root)
res = out.decode().strip().splitlines()
self.assertEqual(len(res), 2)

# get_temp_dir() with a TMPDIR that has a too large length ends up
# NOT being used as it is not possible to use it. In particular,
# the corresponding path will fallback to one of the "built-in"
# ones such as '/tmp', '/var/tmp', or '/usr/tmp'.
root_regex = "|".join(map(re.escape, [root, "/tmp", "/var/tmp", "/usr/tmp"]))

temp_pymp = res[0]
self.assertLess(len(temp_pymp), _SUN_PATH_MAX)
temp_pymp_regex = os.path.join(root_regex, r"pymp-\w{8}")
self.assertRegex(temp_pymp, temp_pymp_regex)

temp_sock = res[1]
self.assertLess(len(temp_sock), _SUN_PATH_MAX)
temp_sock_regex = os.path.join(temp_pymp_regex, r"sock-[0-9a-fA-F]{12}")
self.assertRegex(temp_sock, temp_sock_regex)


if __name__ == '__main__':
unittest.main()
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
:mod:`multiprocessing`: fix creation of temporary socket files following
:gh:`137335` (`PR#148578 <https://github.com/python/cpython/pull/148578>`__
and corresponding backports). Patch by Bénédikt Tran.
Loading

Back | FazBrowse Home | New Git URL