| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent cde19e5 commit 39bc49e
6 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1211,22 +1211,31 @@ Miscellaneous | |||
| 1211 | 1211 | .. versionchanged:: 3.11 | |
| 1212 | 1212 | Accepts a :term:`path-like object`. | |
| 1213 | 1213 | ||
| 1214 | - .. function:: set_forkserver_preload(module_names) | ||
| 1214 | + .. function:: set_forkserver_preload(module_names, *, raise_exceptions=False) | ||
| 1215 | 1215 | ||
| 1216 | 1216 | Set a list of module names for the forkserver main process to attempt to | |
| 1217 | 1217 | import so that their already imported state is inherited by forked | |
| 1218 | - processes. Any :exc:`ImportError` when doing so is silently ignored. | ||
| 1219 | - This can be used as a performance enhancement to avoid repeated work | ||
| 1220 | - in every process. | ||
| 1218 | + processes. This can be used as a performance enhancement to avoid repeated | ||
| 1219 | + work in every process. | ||
| 1221 | 1220 | ||
| 1222 | 1221 | For this to work, it must be called before the forkserver process has been | |
| 1223 | 1222 | launched (before creating a :class:`Pool` or starting a :class:`Process`). | |
| 1224 | 1223 | ||
| 1224 | + By default, any :exc:`ImportError` when importing modules is silently | ||
| 1225 | + ignored. If *raise_exceptions* is ``True``, :exc:`ImportError` exceptions | ||
| 1226 | + will be raised in the forkserver subprocess, causing it to exit. The | ||
| 1227 | + exception traceback will appear on stderr, and subsequent attempts to | ||
| 1228 | + create processes will fail with :exc:`EOFError` or :exc:`ConnectionError`. | ||
| 1229 | + Use *raise_exceptions* during development to catch import problems early. | ||
| 1230 | + | ||
| 1225 | 1231 | Only meaningful when using the ``'forkserver'`` start method. | |
| 1226 | 1232 | See :ref:`multiprocessing-start-methods`. | |
| 1227 | 1233 | ||
| 1228 | 1234 | .. versionadded:: 3.4 | |
| 1229 | 1235 | ||
| 1236 | + .. versionchanged:: next | ||
| 1237 | + Added the *raise_exceptions* parameter. | ||
| 1238 | + | ||
| 1230 | 1239 | .. function:: set_start_method(method, force=False) | |
| 1231 | 1240 | ||
| 1232 | 1241 | Set the method which should be used to start child processes. | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -177,12 +177,15 @@ def set_executable(self, executable): | |||
| 177 | 177 | from .spawn import set_executable | |
| 178 | 178 | set_executable(executable) | |
| 179 | 179 | ||
| 180 | - def set_forkserver_preload(self, module_names): | ||
| 180 | + def set_forkserver_preload(self, module_names, *, raise_exceptions=False): | ||
| 181 | 181 | '''Set list of module names to try to load in forkserver process. | |
| 182 | - This is really just a hint. | ||
| 182 | + | ||
| 183 | + If raise_exceptions is True, ImportError exceptions during preload | ||
| 184 | + will be raised instead of being silently ignored. Such errors will | ||
| 185 | + break all use of the forkserver multiprocessing context. | ||
| 183 | 186 | ''' | |
| 184 | 187 | from .forkserver import set_forkserver_preload | |
| 185 | - set_forkserver_preload(module_names) | ||
| 188 | + set_forkserver_preload(module_names, raise_exceptions=raise_exceptions) | ||
| 186 | 189 | ||
| 187 | 190 | def get_context(self, method=None): | |
| 188 | 191 | if method is None: | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -42,6 +42,7 @@ def __init__(self): | |||
| 42 | 42 | self._inherited_fds = None | |
| 43 | 43 | self._lock = threading.Lock() | |
| 44 | 44 | self._preload_modules = ['__main__'] | |
| 45 | + self._raise_exceptions = False | ||
| 45 | 46 | ||
| 46 | 47 | def _stop(self): | |
| 47 | 48 | # Method used by unit tests to stop the server | |
@@ -64,11 +65,17 @@ def _stop_unlocked(self): | |||
| 64 | 65 | self._forkserver_address = None | |
| 65 | 66 | self._forkserver_authkey = None | |
| 66 | 67 | ||
| 67 | - def set_forkserver_preload(self, modules_names): | ||
| 68 | - '''Set list of module names to try to load in forkserver process.''' | ||
| 68 | + def set_forkserver_preload(self, modules_names, *, raise_exceptions=False): | ||
| 69 | + '''Set list of module names to try to load in forkserver process. | ||
| 70 | + | ||
| 71 | + If raise_exceptions is True, ImportError exceptions during preload | ||
| 72 | + will be raised instead of being silently ignored. Such errors will | ||
| 73 | + break all use of the forkserver multiprocessing context. | ||
| 74 | + ''' | ||
| 69 | 75 | if not all(type(mod) is str for mod in modules_names): | |
| 70 | 76 | raise TypeError('module_names must be a list of strings') | |
| 71 | 77 | self._preload_modules = modules_names | |
| 78 | + self._raise_exceptions = raise_exceptions | ||
| 72 | 79 | ||
| 73 | 80 | def get_inherited_fds(self): | |
| 74 | 81 | '''Return list of fds inherited from parent process. | |
@@ -152,6 +159,8 @@ def ensure_running(self): | |||
| 152 | 159 | main_kws['sys_path'] = data['sys_path'] | |
| 153 | 160 | if 'init_main_from_path' in data: | |
| 154 | 161 | main_kws['main_path'] = data['init_main_from_path'] | |
| 162 | + if self._raise_exceptions: | ||
| 163 | + main_kws['raise_exceptions'] = True | ||
| 155 | 164 | ||
| 156 | 165 | with socket.socket(socket.AF_UNIX) as listener: | |
| 157 | 166 | address = connection.arbitrary_address('AF_UNIX') | |
@@ -197,7 +206,7 @@ def ensure_running(self): | |||
| 197 | 206 | # | |
| 198 | 207 | ||
| 199 | 208 | def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, | |
| 200 | - *, authkey_r=None): | ||
| 209 | + *, authkey_r=None, raise_exceptions=False): | ||
| 201 | 210 | """Run forkserver.""" | |
| 202 | 211 | if authkey_r is not None: | |
| 203 | 212 | try: | |
@@ -221,6 +230,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, | |||
| 221 | 230 | try: | |
| 222 | 231 | __import__(modname) | |
| 223 | 232 | except ImportError: | |
| 233 | + if raise_exceptions: | ||
| 234 | + raise | ||
| 224 | 235 | pass | |
| 225 | 236 | ||
| 226 | 237 | # gh-135335: flush stdout/stderr in case any of the preloaded modules | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,96 @@ | |||
| 1 | + """Tests for forkserver preload functionality.""" | ||
| 2 | + | ||
| 3 | + import multiprocessing | ||
| 4 | + import multiprocessing.forkserver | ||
| 5 | + import sys | ||
| 6 | + import unittest | ||
| 7 | + | ||
| 8 | + | ||
| 9 | + @unittest.skipIf(sys.platform == "win32", "forkserver not available on Windows") | ||
| 10 | + class TestForkserverPreload(unittest.TestCase): | ||
| 11 | + """Tests for forkserver preload functionality.""" | ||
| 12 | + | ||
| 13 | + def setUp(self): | ||
| 14 | + self.ctx = multiprocessing.get_context('forkserver') | ||
| 15 | + # Ensure clean state for each test | ||
| 16 | + multiprocessing.forkserver._forkserver._stop() | ||
| 17 | + | ||
| 18 | + def tearDown(self): | ||
| 19 | + # Clean up after tests | ||
| 20 | + multiprocessing.forkserver._forkserver._stop() | ||
| 21 | + self.ctx.set_forkserver_preload([]) | ||
| 22 | + | ||
| 23 | + @staticmethod | ||
| 24 | + def _send_value(conn, value): | ||
| 25 | + """Helper to send a value through a connection.""" | ||
| 26 | + conn.send(value) | ||
| 27 | + | ||
| 28 | + def test_preload_raise_exceptions_false_default(self): | ||
| 29 | + """Test that invalid modules are silently ignored by default.""" | ||
| 30 | + # With raise_exceptions=False (default), invalid module is ignored | ||
| 31 | + self.ctx.set_forkserver_preload(['nonexistent_module_xyz']) | ||
| 32 | + | ||
| 33 | + # Should be able to start a process without errors | ||
| 34 | + r, w = self.ctx.Pipe(duplex=False) | ||
| 35 | + p = self.ctx.Process(target=self._send_value, args=(w, 42)) | ||
| 36 | + p.start() | ||
| 37 | + w.close() | ||
| 38 | + result = r.recv() | ||
| 39 | + r.close() | ||
| 40 | + p.join() | ||
| 41 | + | ||
| 42 | + self.assertEqual(result, 42) | ||
| 43 | + self.assertEqual(p.exitcode, 0) | ||
| 44 | + | ||
| 45 | + def test_preload_raise_exceptions_false_explicit(self): | ||
| 46 | + """Test that invalid modules are silently ignored with raise_exceptions=False.""" | ||
| 47 | + self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=False) | ||
| 48 | + | ||
| 49 | + # Should be able to start a process without errors | ||
| 50 | + r, w = self.ctx.Pipe(duplex=False) | ||
| 51 | + p = self.ctx.Process(target=self._send_value, args=(w, 99)) | ||
| 52 | + p.start() | ||
| 53 | + w.close() | ||
| 54 | + result = r.recv() | ||
| 55 | + r.close() | ||
| 56 | + p.join() | ||
| 57 | + | ||
| 58 | + self.assertEqual(result, 99) | ||
| 59 | + self.assertEqual(p.exitcode, 0) | ||
| 60 | + | ||
| 61 | + def test_preload_raise_exceptions_true_breaks_context(self): | ||
| 62 | + """Test that invalid modules with raise_exceptions=True breaks the forkserver.""" | ||
| 63 | + self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=True) | ||
| 64 | + | ||
| 65 | + # The forkserver should fail to start when it tries to import | ||
| 66 | + # The exception is raised during p.start() when trying to communicate | ||
| 67 | + # with the failed forkserver process | ||
| 68 | + r, w = self.ctx.Pipe(duplex=False) | ||
| 69 | + try: | ||
| 70 | + p = self.ctx.Process(target=self._send_value, args=(w, 42)) | ||
| 71 | + with self.assertRaises((EOFError, ConnectionError, BrokenPipeError)): | ||
| 72 | + p.start() # Exception raised here | ||
| 73 | + finally: | ||
| 74 | + # Ensure pipes are closed even if exception is raised | ||
| 75 | + w.close() | ||
| 76 | + r.close() | ||
| 77 | + | ||
| 78 | + def test_preload_valid_modules_with_raise_exceptions(self): | ||
| 79 | + """Test that valid modules work fine with raise_exceptions=True.""" | ||
| 80 | + # Valid modules should work even with raise_exceptions=True | ||
| 81 | + self.ctx.set_forkserver_preload(['os', 'sys'], raise_exceptions=True) | ||
| 82 | + | ||
| 83 | + r, w = self.ctx.Pipe(duplex=False) | ||
| 84 | + p = self.ctx.Process(target=self._send_value, args=(w, 'success')) | ||
| 85 | + p.start() | ||
| 86 | + w.close() | ||
| 87 | + result = r.recv() | ||
| 88 | + r.close() | ||
| 89 | + p.join() | ||
| 90 | + | ||
| 91 | + self.assertEqual(result, 'success') | ||
| 92 | + self.assertEqual(p.exitcode, 0) | ||
| 93 | + | ||
| 94 | + | ||
| 95 | + if __name__ == '__main__': | ||
| 96 | + unittest.main() | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1337,6 +1337,7 @@ Trent Nelson | |||
| 1337 | 1337 | Andrew Nester | |
| 1338 | 1338 | Osvaldo Santana Neto | |
| 1339 | 1339 | Chad Netzer | |
| 1340 | + Nick Neumann | ||
| 1340 | 1341 | Max Neunhöffer | |
| 1341 | 1342 | Anthon van der Neut | |
| 1342 | 1343 | George Neville-Neil | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,5 @@ | |||
| 1 | + Add a ``raise_exceptions`` keyword-only parameter to | ||
| 2 | + :func:`multiprocessing.set_forkserver_preload`. When set to ``True``, | ||
| 3 | + :exc:`ImportError` exceptions during module preloading will be raised instead | ||
| 4 | + of being silently ignored. Defaults to ``False`` for backward compatibility. | ||
| 5 | + Contributed by Nick Neumann. | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments