| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,143 @@ | |||
| 1 | + using NUnit.Framework; | ||
| 2 | + using Python.Runtime; | ||
| 3 | + using System; | ||
| 4 | + using System.Threading; | ||
| 5 | + | ||
| 6 | + namespace Python.EmbeddingTest | ||
| 7 | + { | ||
| 8 | + public class TestFinalizer | ||
| 9 | + { | ||
| 10 | + private string _PYTHONMALLOC = string.Empty; | ||
| 11 | + | ||
| 12 | + [SetUp] | ||
| 13 | + public void SetUp() | ||
| 14 | + { | ||
| 15 | + try | ||
| 16 | + { | ||
| 17 | + _PYTHONMALLOC = Environment.GetEnvironmentVariable("PYTHONMALLOC"); | ||
| 18 | + } | ||
| 19 | + catch (ArgumentNullException) | ||
| 20 | + { | ||
| 21 | + _PYTHONMALLOC = string.Empty; | ||
| 22 | + } | ||
| 23 | + Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); | ||
| 24 | + PythonEngine.Initialize(); | ||
| 25 | + } | ||
| 26 | + | ||
| 27 | + [TearDown] | ||
| 28 | + public void TearDown() | ||
| 29 | + { | ||
| 30 | + PythonEngine.Shutdown(); | ||
| 31 | + if (string.IsNullOrEmpty(_PYTHONMALLOC)) | ||
| 32 | + { | ||
| 33 | + Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); | ||
| 34 | + } | ||
| 35 | + } | ||
| 36 | + | ||
| 37 | + private static void FullGCCollect() | ||
| 38 | + { | ||
| 39 | + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); | ||
| 40 | + GC.WaitForFullGCComplete(); | ||
| 41 | + GC.WaitForPendingFinalizers(); | ||
| 42 | + } | ||
| 43 | + | ||
| 44 | + [Test] | ||
| 45 | + public void CollectBasicObject() | ||
| 46 | + { | ||
| 47 | + int thId = Thread.CurrentThread.ManagedThreadId; | ||
| 48 | + Finalizer.Instance.Threshold = 1; | ||
| 49 | + bool called = false; | ||
| 50 | + EventHandler<Finalizer.CollectArgs> handler = (s, e) => | ||
| 51 | + { | ||
| 52 | + Assert.AreEqual(thId, Thread.CurrentThread.ManagedThreadId); | ||
| 53 | + Assert.GreaterOrEqual(e.ObjectCount, 1); | ||
| 54 | + called = true; | ||
| 55 | + }; | ||
| 56 | + Finalizer.Instance.CollectOnce += handler; | ||
| 57 | + FullGCCollect(); | ||
| 58 | + PyLong obj = new PyLong(1024); | ||
| 59 | + | ||
| 60 | + WeakReference shortWeak = new WeakReference(obj); | ||
| 61 | + WeakReference longWeak = new WeakReference(obj, true); | ||
| 62 | + obj = null; | ||
| 63 | + FullGCCollect(); | ||
| 64 | + // The object has been resurrected | ||
| 65 | + Assert.IsFalse(shortWeak.IsAlive); | ||
| 66 | + Assert.IsTrue(longWeak.IsAlive); | ||
| 67 | + | ||
| 68 | + Assert.IsFalse(called); | ||
| 69 | + var garbage = Finalizer.Instance.GetCollectedObjects(); | ||
| 70 | + // FIXME: If make some query for garbage, | ||
| 71 | + // the above case will failed Assert.IsFalse(shortWeak.IsAlive) | ||
| 72 | + //Assert.IsTrue(garbage.All(T => T.IsAlive)); | ||
| 73 | + | ||
| 74 | + Finalizer.Instance.CallPendingFinalizers(); | ||
| 75 | + Assert.IsTrue(called); | ||
| 76 | + | ||
| 77 | + FullGCCollect(); | ||
| 78 | + //Assert.IsFalse(garbage.All(T => T.IsAlive)); | ||
| 79 | + | ||
| 80 | + Assert.IsNull(longWeak.Target); | ||
| 81 | + | ||
| 82 | + Finalizer.Instance.CollectOnce -= handler; | ||
| 83 | + } | ||
| 84 | + | ||
| 85 | + private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) | ||
| 86 | + { | ||
| 87 | + // Must larger than 512 bytes make sure Python use | ||
| 88 | + string str = new string('1', 1024); | ||
| 89 | + Finalizer.Instance.Enable = true; | ||
| 90 | + FullGCCollect(); | ||
| 91 | + FullGCCollect(); | ||
| 92 | + pyCollect.Invoke(); | ||
| 93 | + Finalizer.Instance.CallPendingFinalizers(); | ||
| 94 | + Finalizer.Instance.Enable = enbale; | ||
| 95 | + | ||
| 96 | + // Estimate unmanaged memory size | ||
| 97 | + long before = Environment.WorkingSet - GC.GetTotalMemory(true); | ||
| 98 | + for (int i = 0; i < 10000; i++) | ||
| 99 | + { | ||
| 100 | + // Memory will leak when disable Finalizer | ||
| 101 | + new PyString(str); | ||
| 102 | + } | ||
| 103 | + FullGCCollect(); | ||
| 104 | + FullGCCollect(); | ||
| 105 | + pyCollect.Invoke(); | ||
| 106 | + if (enbale) | ||
| 107 | + { | ||
| 108 | + Finalizer.Instance.CallPendingFinalizers(); | ||
| 109 | + } | ||
| 110 | + | ||
| 111 | + FullGCCollect(); | ||
| 112 | + FullGCCollect(); | ||
| 113 | + long after = Environment.WorkingSet - GC.GetTotalMemory(true); | ||
| 114 | + return after - before; | ||
| 115 | + | ||
| 116 | + } | ||
| 117 | + | ||
| 118 | + /// <summary> | ||
| 119 | + /// Because of two vms both have their memory manager, | ||
| 120 | + /// this test only prove the finalizer has take effect. | ||
| 121 | + /// </summary> | ||
| 122 | + [Test] | ||
| 123 | + [Ignore("Too many uncertainties, only manual on when debugging")] | ||
| 124 | + public void SimpleTestMemory() | ||
| 125 | + { | ||
| 126 | + bool oldState = Finalizer.Instance.Enable; | ||
| 127 | + try | ||
| 128 | + { | ||
| 129 | + using (PyObject gcModule = PythonEngine.ImportModule("gc")) | ||
| 130 | + using (PyObject pyCollect = gcModule.GetAttr("collect")) | ||
| 131 | + { | ||
| 132 | + long span1 = CompareWithFinalizerOn(pyCollect, false); | ||
| 133 | + long span2 = CompareWithFinalizerOn(pyCollect, true); | ||
| 134 | + Assert.Less(span2, span1); | ||
| 135 | + } | ||
| 136 | + } | ||
| 137 | + finally | ||
| 138 | + { | ||
| 139 | + Finalizer.Instance.Enable = oldState; | ||
| 140 | + } | ||
| 141 | + } | ||
| 142 | + } | ||
| 143 | + } | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -181,10 +181,12 @@ A possible alternate strategy would be to create custom subclasses | |||
| 181 | 181 | too "special" for this to work. It would be more work, so for now | |
| 182 | 182 | the 80/20 rule applies :) */ | |
| 183 | 183 | ||
| 184 | - public class Dispatcher | ||
| 184 | + public class Dispatcher : IDisposable | ||
| 185 | 185 | { | |
| 186 | 186 | public IntPtr target; | |
| 187 | 187 | public Type dtype; | |
| 188 | + private bool _disposed = false; | ||
| 189 | + private bool _finalized = false; | ||
| 188 | 190 | ||
| 189 | 191 | public Dispatcher(IntPtr target, Type dtype) | |
| 190 | 192 | { | |
@@ -195,18 +197,25 @@ public Dispatcher(IntPtr target, Type dtype) | |||
| 195 | 197 | ||
| 196 | 198 | ~Dispatcher() | |
| 197 | 199 | { | |
| 198 | - // We needs to disable Finalizers until it's valid implementation. | ||
| 199 | - // Current implementation can produce low probability floating bugs. | ||
| 200 | - return; | ||
| 200 | + if (_finalized || _disposed) | ||
| 201 | + { | ||
| 202 | + return; | ||
| 203 | + } | ||
| 204 | + _finalized = true; | ||
| 205 | + Finalizer.Instance.AddFinalizedObject(this); | ||
| 206 | + } | ||
| 201 | 207 | ||
| 202 | - // Note: the managed GC thread can run and try to free one of | ||
| 203 | - // these *after* the Python runtime has been finalized! | ||
| 204 | - if (Runtime.Py_IsInitialized() > 0) | ||
| 208 | + public void Dispose() | ||
| 209 | + { | ||
| 210 | + if (_disposed) | ||
| 205 | 211 | { | |
| 206 | - IntPtr gs = PythonEngine.AcquireLock(); | ||
| 207 | - Runtime.XDecref(target); | ||
| 208 | - PythonEngine.ReleaseLock(gs); | ||
| 212 | + return; | ||
| 209 | 213 | } | |
| 214 | + _disposed = true; | ||
| 215 | + Runtime.XDecref(target); | ||
| 216 | + target = IntPtr.Zero; | ||
| 217 | + dtype = null; | ||
| 218 | + GC.SuppressFinalize(this); | ||
| 210 | 219 | } | |
| 211 | 220 | ||
| 212 | 221 | public object Dispatch(ArrayList args) | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,117 @@ | |||
| 1 | + using System; | ||
| 2 | + using System.Collections.Concurrent; | ||
| 3 | + using System.Collections.Generic; | ||
| 4 | + using System.Linq; | ||
| 5 | + using System.Runtime.InteropServices; | ||
| 6 | + using System.Threading; | ||
| 7 | + | ||
| 8 | + namespace Python.Runtime | ||
| 9 | + { | ||
| 10 | + public class Finalizer | ||
| 11 | + { | ||
| 12 | + public class CollectArgs : EventArgs | ||
| 13 | + { | ||
| 14 | + public int ObjectCount { get; set; } | ||
| 15 | + } | ||
| 16 | + | ||
| 17 | + public static readonly Finalizer Instance = new Finalizer(); | ||
| 18 | + | ||
| 19 | + public event EventHandler<CollectArgs> CollectOnce; | ||
| 20 | + | ||
| 21 | + private ConcurrentQueue<IDisposable> _objQueue = new ConcurrentQueue<IDisposable>(); | ||
| 22 | + | ||
| 23 | + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] | ||
| 24 | + private delegate int PedingCall(IntPtr arg); | ||
| 25 | + private PedingCall _collectAction; | ||
| 26 | + private bool _pending = false; | ||
| 27 | + private readonly object _collectingLock = new object(); | ||
| 28 | + public int Threshold { get; set; } | ||
| 29 | + public bool Enable { get; set; } | ||
| 30 | + | ||
| 31 | + private Finalizer() | ||
| 32 | + { | ||
| 33 | + Enable = true; | ||
| 34 | + Threshold = 200; | ||
| 35 | + _collectAction = OnCollect; | ||
| 36 | + } | ||
| 37 | + | ||
| 38 | + public void CallPendingFinalizers() | ||
| 39 | + { | ||
| 40 | + if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) | ||
| 41 | + { | ||
| 42 | + throw new Exception("PendingCall should execute in main Python thread"); | ||
| 43 | + } | ||
| 44 | + Runtime.Py_MakePendingCalls(); | ||
| 45 | + } | ||
| 46 | + | ||
| 47 | + public List<WeakReference> GetCollectedObjects() | ||
| 48 | + { | ||
| 49 | + return _objQueue.Select(T => new WeakReference(T)).ToList(); | ||
| 50 | + } | ||
| 51 | + | ||
| 52 | + internal void AddFinalizedObject(IDisposable obj) | ||
| 53 | + { | ||
| 54 | + if (!Enable) | ||
| 55 | + { | ||
| 56 | + return; | ||
| 57 | + } | ||
| 58 | + if (Runtime.Py_IsInitialized() == 0) | ||
| 59 | + { | ||
| 60 | + // XXX: Memory will leak if a PyObject finalized after Python shutdown, | ||
| 61 | + // for avoiding that case, user should call GC.Collect manual before shutdown. | ||
| 62 | + return; | ||
| 63 | + } | ||
| 64 | + _objQueue.Enqueue(obj); | ||
| 65 | + GC.ReRegisterForFinalize(obj); | ||
| 66 | + if (_objQueue.Count >= Threshold) | ||
| 67 | + { | ||
| 68 | + Collect(); | ||
| 69 | + } | ||
| 70 | + } | ||
| 71 | + | ||
| 72 | + internal static void Shutdown() | ||
| 73 | + { | ||
| 74 | + Instance.DisposeAll(); | ||
| 75 | + Instance.CallPendingFinalizers(); | ||
| 76 | + Runtime.PyErr_Clear(); | ||
| 77 | + } | ||
| 78 | + | ||
| 79 | + private void Collect() | ||
| 80 | + { | ||
| 81 | + lock (_collectingLock) | ||
| 82 | + { | ||
| 83 | + if (_pending) | ||
| 84 | + { | ||
| 85 | + return; | ||
| 86 | + } | ||
| 87 | + _pending = true; | ||
| 88 | + } | ||
| 89 | + IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); | ||
| 90 | + if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) | ||
| 91 | + { | ||
| 92 | + // Full queue, append next time | ||
| 93 | + _pending = false; | ||
| 94 | + } | ||
| 95 | + } | ||
| 96 | + | ||
| 97 | + private int OnCollect(IntPtr arg) | ||
| 98 | + { | ||
| 99 | + CollectOnce?.Invoke(this, new CollectArgs() | ||
| 100 | + { | ||
| 101 | + ObjectCount = _objQueue.Count | ||
| 102 | + }); | ||
| 103 | + DisposeAll(); | ||
| 104 | + _pending = false; | ||
| 105 | + return 0; | ||
| 106 | + } | ||
| 107 | + | ||
| 108 | + private void DisposeAll() | ||
| 109 | + { | ||
| 110 | + IDisposable obj; | ||
| 111 | + while (_objQueue.TryDequeue(out obj)) | ||
| 112 | + { | ||
| 113 | + obj.Dispose(); | ||
| 114 | + } | ||
| 115 | + } | ||
| 116 | + } | ||
| 117 | + } | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -17,6 +17,7 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable | |||
| 17 | 17 | { | |
| 18 | 18 | protected internal IntPtr obj = IntPtr.Zero; | |
| 19 | 19 | private bool disposed = false; | |
| 20 | + private bool _finalized = false; | ||
| 20 | 21 | ||
| 21 | 22 | /// <summary> | |
| 22 | 23 | /// PyObject Constructor | |
@@ -41,14 +42,15 @@ protected PyObject() | |||
| 41 | 42 | ||
| 42 | 43 | // Ensure that encapsulated Python object is decref'ed appropriately | |
| 43 | 44 | // when the managed wrapper is garbage-collected. | |
| 44 | - | ||
| 45 | 45 | ~PyObject() | |
| 46 | 46 | { | |
| 47 | - // We needs to disable Finalizers until it's valid implementation. | ||
| 48 | - // Current implementation can produce low probability floating bugs. | ||
| 49 | - return; | ||
| 50 | - | ||
| 51 | - Dispose(); | ||
| 47 | + if (_finalized || disposed) | ||
| 48 | + { | ||
| 49 | + return; | ||
| 50 | + } | ||
| 51 | + // Prevent a infinity loop by calling GC.WaitForPendingFinalizers | ||
| 52 | + _finalized = true; | ||
| 53 | + Finalizer.Instance.AddFinalizedObject(this); | ||
| 52 | 54 | } | |
| 53 | 55 | ||
| 54 | 56 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -37,6 +37,7 @@ public class PyScope : DynamicObject, IDisposable | |||
| 37 | 37 | internal readonly IntPtr variables; | |
| 38 | 38 | ||
| 39 | 39 | private bool _isDisposed; | |
| 40 | + private bool _finalized = false; | ||
| 40 | 41 | ||
| 41 | 42 | /// <summary> | |
| 42 | 43 | /// The Manager this scope associated with. | |
@@ -527,11 +528,12 @@ public void Dispose() | |||
| 527 | 528 | ||
| 528 | 529 | ~PyScope() | |
| 529 | 530 | { | |
| 530 | - // We needs to disable Finalizers until it's valid implementation. | ||
| 531 | - // Current implementation can produce low probability floating bugs. | ||
| 532 | - return; | ||
| 533 | - | ||
| 534 | - Dispose(); | ||
| 531 | + if (_finalized || _isDisposed) | ||
| 532 | + { | ||
| 533 | + return; | ||
| 534 | + } | ||
| 535 | + _finalized = true; | ||
| 536 | + Finalizer.Instance.AddFinalizedObject(this); | ||
| 535 | 537 | } | |
| 536 | 538 | } | |
| 537 | 539 | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments