| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 5fb443d commit a5b1bdf
3 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -167,6 +167,8 @@ def ensure_running(self): | |||
| 167 | 167 | def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): | |
| 168 | 168 | '''Run forkserver.''' | |
| 169 | 169 | if preload: | |
| 170 | + if sys_path is not None: | ||
| 171 | + sys.path[:] = sys_path | ||
| 170 | 172 | if '__main__' in preload and main_path is not None: | |
| 171 | 173 | process.current_process()._inheriting = True | |
| 172 | 174 | try: | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -12,6 +12,7 @@ | |||
| 12 | 12 | import sys | |
| 13 | 13 | import os | |
| 14 | 14 | import gc | |
| 15 | + import importlib | ||
| 15 | 16 | import errno | |
| 16 | 17 | import functools | |
| 17 | 18 | import signal | |
@@ -20,8 +21,10 @@ | |||
| 20 | 21 | import socket | |
| 21 | 22 | import random | |
| 22 | 23 | import logging | |
| 24 | + import shutil | ||
| 23 | 25 | import subprocess | |
| 24 | 26 | import struct | |
| 27 | + import tempfile | ||
| 25 | 28 | import operator | |
| 26 | 29 | import pickle | |
| 27 | 30 | import weakref | |
@@ -6196,6 +6199,81 @@ def submain(): pass | |||
| 6196 | 6199 | self.assertFalse(err, msg=err.decode('utf-8')) | |
| 6197 | 6200 | ||
| 6198 | 6201 | ||
| 6202 | + class _TestSpawnedSysPath(BaseTestCase): | ||
| 6203 | + """Test that sys.path is setup in forkserver and spawn processes.""" | ||
| 6204 | + | ||
| 6205 | + ALLOWED_TYPES = ('processes',) | ||
| 6206 | + | ||
| 6207 | + def setUp(self): | ||
| 6208 | + self._orig_sys_path = list(sys.path) | ||
| 6209 | + self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-") | ||
| 6210 | + self._mod_name = "unique_test_mod" | ||
| 6211 | + module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py") | ||
| 6212 | + with open(module_path, "w", encoding="utf-8") as mod: | ||
| 6213 | + mod.write("# A simple test module\n") | ||
| 6214 | + sys.path[:] = [p for p in sys.path if p] # remove any existing ""s | ||
| 6215 | + sys.path.insert(0, self._temp_dir) | ||
| 6216 | + sys.path.insert(0, "") # Replaced with an abspath in child. | ||
| 6217 | + try: | ||
| 6218 | + self._ctx_forkserver = multiprocessing.get_context("forkserver") | ||
| 6219 | + except ValueError: | ||
| 6220 | + self._ctx_forkserver = None | ||
| 6221 | + self._ctx_spawn = multiprocessing.get_context("spawn") | ||
| 6222 | + | ||
| 6223 | + def tearDown(self): | ||
| 6224 | + sys.path[:] = self._orig_sys_path | ||
| 6225 | + shutil.rmtree(self._temp_dir, ignore_errors=True) | ||
| 6226 | + | ||
| 6227 | + @staticmethod | ||
| 6228 | + def enq_imported_module_names(queue): | ||
| 6229 | + queue.put(tuple(sys.modules)) | ||
| 6230 | + | ||
| 6231 | + def test_forkserver_preload_imports_sys_path(self): | ||
| 6232 | + ctx = self._ctx_forkserver | ||
| 6233 | + if not ctx: | ||
| 6234 | + self.skipTest("requires forkserver start method.") | ||
| 6235 | + self.assertNotIn(self._mod_name, sys.modules) | ||
| 6236 | + multiprocessing.forkserver._forkserver._stop() # Must be fresh. | ||
| 6237 | + ctx.set_forkserver_preload( | ||
| 6238 | + ["test.test_multiprocessing_forkserver", self._mod_name]) | ||
| 6239 | + q = ctx.Queue() | ||
| 6240 | + proc = ctx.Process(target=self.enq_imported_module_names, args=(q,)) | ||
| 6241 | + proc.start() | ||
| 6242 | + proc.join() | ||
| 6243 | + child_imported_modules = q.get() | ||
| 6244 | + q.close() | ||
| 6245 | + self.assertIn(self._mod_name, child_imported_modules) | ||
| 6246 | + | ||
| 6247 | + @staticmethod | ||
| 6248 | + def enq_sys_path_and_import(queue, mod_name): | ||
| 6249 | + queue.put(sys.path) | ||
| 6250 | + try: | ||
| 6251 | + importlib.import_module(mod_name) | ||
| 6252 | + except ImportError as exc: | ||
| 6253 | + queue.put(exc) | ||
| 6254 | + else: | ||
| 6255 | + queue.put(None) | ||
| 6256 | + | ||
| 6257 | + def test_child_sys_path(self): | ||
| 6258 | + for ctx in (self._ctx_spawn, self._ctx_forkserver): | ||
| 6259 | + if not ctx: | ||
| 6260 | + continue | ||
| 6261 | + with self.subTest(f"{ctx.get_start_method()} start method"): | ||
| 6262 | + q = ctx.Queue() | ||
| 6263 | + proc = ctx.Process(target=self.enq_sys_path_and_import, | ||
| 6264 | + args=(q, self._mod_name)) | ||
| 6265 | + proc.start() | ||
| 6266 | + proc.join() | ||
| 6267 | + child_sys_path = q.get() | ||
| 6268 | + import_error = q.get() | ||
| 6269 | + q.close() | ||
| 6270 | + self.assertNotIn("", child_sys_path) # replaced by an abspath | ||
| 6271 | + self.assertIn(self._temp_dir, child_sys_path) # our addition | ||
| 6272 | + # ignore the first element, it is the absolute "" replacement | ||
| 6273 | + self.assertEqual(child_sys_path[1:], sys.path[1:]) | ||
| 6274 | + self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}") | ||
| 6275 | + | ||
| 6276 | + | ||
| 6199 | 6277 | class MiscTestCase(unittest.TestCase): | |
| 6200 | 6278 | def test__all__(self): | |
| 6201 | 6279 | # Just make sure names in not_exported are excluded | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,17 @@ | |||
| 1 | + Fixed the :mod:`multiprocessing` ``"forkserver"`` start method forkserver | ||
| 2 | + process to correctly inherit the parent's :data:`sys.path` during the importing | ||
| 3 | + of :func:`multiprocessing.set_forkserver_preload` modules in the same manner as | ||
| 4 | + :data:`sys.path` is configured in workers before executing work items. | ||
| 5 | + | ||
| 6 | + This bug caused some forkserver module preloading to silently fail to preload. | ||
| 7 | + This manifested as a performance degration in child processes when the | ||
| 8 | + ``sys.path`` was required due to additional repeated work in every worker. | ||
| 9 | + | ||
| 10 | + It could also have a side effect of ``""`` remaining in :data:`sys.path` during | ||
| 11 | + forkserver preload imports instead of the absolute path from :func:`os.getcwd` | ||
| 12 | + at multiprocessing import time used in the worker ``sys.path``. | ||
| 13 | + | ||
| 14 | + Potentially leading to incorrect imports from the wrong location during | ||
| 15 | + preload. We are unaware of that actually happening. The issue was discovered | ||
| 16 | + by someone observing unexpected preload performance gains. | ||
| 17 | + | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments