| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 0768eed commit 48ddda1
4 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -160,11 +160,21 @@ to start a process. These *start methods* are | |||
| 160 | 160 | No unnecessary resources are inherited. | |
| 161 | 161 | ||
| 162 | 162 | Available on POSIX platforms which support passing file descriptors over | |
| 163 | - Unix pipes such as Linux. The default on those. | ||
| 163 | + Unix pipes such as Linux. The default on those, unless there is no | ||
| 164 | + writable temporary directory (see :func:`tempfile.gettempdir`), which | ||
| 165 | + the fork server requires; *spawn* is the default in that case. | ||
| 164 | 166 | ||
| 165 | 167 | .. versionchanged:: 3.14 | |
| 166 | 168 | This became the default start method on POSIX platforms. | |
| 167 | 169 | ||
| 170 | + .. versionchanged:: next | ||
| 171 | + *spawn* is now the default start method when there is no writable | ||
| 172 | + temporary directory, for example on a read-only file system. | ||
| 173 | + Previously *forkserver* remained the default there and starting a | ||
| 174 | + process failed with an :exc:`OSError`. The default is determined | ||
| 175 | + when it is first needed rather than when :mod:`!multiprocessing` | ||
| 176 | + is imported. See :gh:`155717`. | ||
| 177 | + | ||
| 168 | 178 | .. versionchanged:: 3.4 | |
| 169 | 179 | *spawn* added on all POSIX platforms, and *forkserver* added for | |
| 170 | 180 | some POSIX platforms. | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,10 +1,10 @@ | |||
| 1 | + import functools | ||
| 1 | 2 | import os | |
| 2 | 3 | import sys | |
| 3 | 4 | import threading | |
| 4 | 5 | ||
| 5 | 6 | from . import process | |
| 6 | 7 | from . import reduction | |
| 7 | - from . import util | ||
| 8 | 8 | ||
| 9 | 9 | __all__ = () | |
| 10 | 10 | ||
@@ -240,10 +240,19 @@ def _after_fork(): | |||
| 240 | 240 | class DefaultContext(BaseContext): | |
| 241 | 241 | Process = Process | |
| 242 | 242 | ||
| 243 | - def __init__(self, context): | ||
| 244 | - self._default_context = context | ||
| 243 | + def __init__(self, context=None): | ||
| 244 | + if context is not None: | ||
| 245 | + self._default_context = context | ||
| 245 | 246 | self._actual_context = None | |
| 246 | 247 | ||
| 248 | + # The platform default is only determined when first needed, as doing | ||
| 249 | + # so can be costly; importing multiprocessing must stay cheap. | ||
| 250 | + # A cached_property rather than a property so that the attribute can | ||
| 251 | + # still be assigned, deleted and patched like a plain one. | ||
| 252 | + @functools.cached_property | ||
| 253 | + def _default_context(self): | ||
| 254 | + return _get_platform_default_context() | ||
| 255 | + | ||
| 247 | 256 | def get_context(self, method=None): | |
| 248 | 257 | if method is None: | |
| 249 | 258 | if self._actual_context is None: | |
@@ -331,18 +340,21 @@ def _check_available(self): | |||
| 331 | 340 | if reduction.HAVE_SEND_HANDLE: | |
| 332 | 341 | _concrete_contexts['forkserver'] = ForkServerContext() | |
| 333 | 342 | ||
| 334 | - # bpo-33725: running arbitrary code after fork() is no longer reliable | ||
| 335 | - # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. | ||
| 336 | - # gh-84559: We changed everyones default to a thread safeish one in 3.14. | ||
| 337 | - if ( | ||
| 338 | - reduction.HAVE_SEND_HANDLE | ||
| 339 | - and sys.platform != 'darwin' | ||
| 340 | - # gh-155717: forkserver requires to write temporary files | ||
| 341 | - and util._has_writeable_tempdir() | ||
| 342 | - ): | ||
| 343 | - _default_context = DefaultContext(_concrete_contexts['forkserver']) | ||
| 344 | - else: | ||
| 345 | - _default_context = DefaultContext(_concrete_contexts['spawn']) | ||
| 343 | + def _get_platform_default_context(): | ||
| 344 | + # bpo-33725: running arbitrary code after fork() is no longer reliable | ||
| 345 | + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. | ||
| 346 | + # gh-84559: We changed everyones default to a thread safeish one in 3.14. | ||
| 347 | + if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': | ||
| 348 | + # gh-155717: forkserver requires to write temporary files. | ||
| 349 | + # | ||
| 350 | + # Imported here rather than at the top level as util is costly | ||
| 351 | + # to import and is not otherwise needed to import this module. | ||
| 352 | + from . import util | ||
| 353 | + if util._has_writeable_tempdir(): | ||
| 354 | + return _concrete_contexts['forkserver'] | ||
| 355 | + return _concrete_contexts['spawn'] | ||
| 356 | + | ||
| 357 | + _default_context = DefaultContext() | ||
| 346 | 358 | ||
| 347 | 359 | else: # Windows | |
| 348 | 360 | ||
@@ -365,7 +377,10 @@ class SpawnContext(BaseContext): | |||
| 365 | 377 | _concrete_contexts = { | |
| 366 | 378 | 'spawn': SpawnContext(), | |
| 367 | 379 | } | |
| 368 | - _default_context = DefaultContext(_concrete_contexts['spawn']) | ||
| 380 | + def _get_platform_default_context(): | ||
| 381 | + return _concrete_contexts['spawn'] | ||
| 382 | + | ||
| 383 | + _default_context = DefaultContext() | ||
| 369 | 384 | ||
| 370 | 385 | # | |
| 371 | 386 | # Force the start method | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -10,7 +10,6 @@ | |||
| 10 | 10 | import os | |
| 11 | 11 | import itertools | |
| 12 | 12 | import sys | |
| 13 | - import tempfile | ||
| 14 | 13 | import weakref | |
| 15 | 14 | import atexit | |
| 16 | 15 | import threading # we want threading to install it's | |
@@ -155,7 +154,7 @@ def _remove_temp_dir(rmtree, tempdir): | |||
| 155 | 154 | current_process._config['tempdir'] = None | |
| 156 | 155 | ||
| 157 | 156 | ||
| 158 | - def _get_base_temp_dir(): | ||
| 157 | + def _get_base_temp_dir(tempfile): | ||
| 159 | 158 | """Get a temporary directory where socket files will be created. | |
| 160 | 159 | ||
| 161 | 160 | To prevent additional imports, pass a pre-imported 'tempfile' module. | |
@@ -216,8 +215,9 @@ def get_temp_dir(): | |||
| 216 | 215 | # get name of a temp directory which will be automatically cleaned up | |
| 217 | 216 | tempdir = process.current_process()._config.get('tempdir') | |
| 218 | 217 | if tempdir is None: | |
| 219 | - import shutil | ||
| 220 | - base_tempdir = _get_base_temp_dir() | ||
| 218 | + # Deferred until first use to keep importing this module cheap. | ||
| 219 | + import shutil, tempfile | ||
| 220 | + base_tempdir = _get_base_temp_dir(tempfile) | ||
| 221 | 221 | tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) | |
| 222 | 222 | info('created temp directory %s', tempdir) | |
| 223 | 223 | # keep a strong reference to shutil.rmtree(), since the finalizer | |
@@ -234,7 +234,14 @@ def _has_writeable_tempdir(): | |||
| 234 | 234 | # | |
| 235 | 235 | # See: https://github.com/python/cpython/issues/155717. | |
| 236 | 236 | ||
| 237 | - path = _get_base_temp_dir() | ||
| 237 | + # Deferred until first use to keep importing this module cheap. | ||
| 238 | + import tempfile | ||
| 239 | + | ||
| 240 | + try: | ||
| 241 | + path = _get_base_temp_dir(tempfile) | ||
| 242 | + except OSError: | ||
| 243 | + # No usable temporary directory at all. | ||
| 244 | + return False | ||
| 238 | 245 | if path is None: | |
| 239 | 246 | return False | |
| 240 | 247 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -6516,6 +6516,62 @@ def test_forkserver_requires_writeable_tempdir(self, mode): | |||
| 6516 | 6516 | _, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR) | |
| 6517 | 6517 | self.assertEqual(out.decode().strip(), "spawn") | |
| 6518 | 6518 | ||
| 6519 | + @unittest.skipIf(os.name == "nt", "requires POSIX") | ||
| 6520 | + @support.requires_non_root_user | ||
| 6521 | + def test_default_start_method_without_usable_tempdir(self): | ||
| 6522 | + # Neither importing multiprocessing nor determining the default | ||
| 6523 | + # start method may fail when tempfile finds no usable temporary | ||
| 6524 | + # directory at all, and the default start method must be 'spawn'. | ||
| 6525 | + # | ||
| 6526 | + # See https://github.com/python/cpython/issues/155717. | ||
| 6527 | + | ||
| 6528 | + cmd = '''if 1: | ||
| 6529 | + import os, tempfile | ||
| 6530 | + # Make TMPDIR the only candidate, so that tempfile.gettempdir() | ||
| 6531 | + # raises FileNotFoundError instead of falling back to /tmp. | ||
| 6532 | + tempfile._candidate_tempdir_list = lambda: [os.environ["TMPDIR"]] | ||
| 6533 | + try: | ||
| 6534 | + tempfile.gettempdir() | ||
| 6535 | + except FileNotFoundError: | ||
| 6536 | + pass | ||
| 6537 | + else: | ||
| 6538 | + raise AssertionError("TMPDIR is usable") | ||
| 6539 | + | ||
| 6540 | + import multiprocessing | ||
| 6541 | + if __name__ == "__main__": | ||
| 6542 | + print(multiprocessing.get_start_method(), | ||
| 6543 | + multiprocessing.get_all_start_methods()[0]) | ||
| 6544 | + ''' | ||
| 6545 | + | ||
| 6546 | + with support.os_helper.temp_dir() as root: | ||
| 6547 | + TMPDIR = pathlib.Path(root, "TMPDIR") | ||
| 6548 | + TMPDIR.mkdir(mode=os.R_OK | os.X_OK) | ||
| 6549 | + _, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR) | ||
| 6550 | + self.assertEqual(out.decode().split(), ["spawn", "spawn"]) | ||
| 6551 | + | ||
| 6552 | + def test_import_does_not_determine_default_start_method(self): | ||
| 6553 | + # Determining the default start method requires checking that the | ||
| 6554 | + # temporary directory is writeable. That must not happen, nor may | ||
| 6555 | + # the modules it needs be loaded, when importing multiprocessing. | ||
| 6556 | + import_helper.ensure_lazy_imports( | ||
| 6557 | + "multiprocessing", {"tempfile", "shutil", "subprocess"}) | ||
| 6558 | + | ||
| 6559 | + cmd = '''if 1: | ||
| 6560 | + import sys | ||
| 6561 | + import multiprocessing | ||
| 6562 | + print(multiprocessing.get_start_method(allow_none=True), | ||
| 6563 | + "tempfile" in sys.modules, | ||
| 6564 | + multiprocessing.get_start_method(), | ||
| 6565 | + multiprocessing.get_all_start_methods()[0]) | ||
| 6566 | + ''' | ||
| 6567 | + # -S: site customizations could import tempfile themselves. | ||
| 6568 | + _, out, err = script_helper.assert_python_ok('-S', '-c', cmd) | ||
| 6569 | + unset, probed, method, first = out.decode().split() | ||
| 6570 | + self.assertEqual(unset, "None") | ||
| 6571 | + self.assertEqual(probed, "False") | ||
| 6572 | + self.assertEqual(method, first) | ||
| 6573 | + self.assertIn(method, multiprocessing.get_all_start_methods()) | ||
| 6574 | + | ||
| 6519 | 6575 | ||
| 6520 | 6576 | @unittest.skipIf(sys.platform == "win32", | |
| 6521 | 6577 | "test semantics don't make sense on Windows") | |
| Back | FazBrowse Home | New Git URL |
0 commit comments