| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 9b8dcc6 commit 6779652
5 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -102,6 +102,7 @@ PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); | |||
| 102 | 102 | Py_ssize_t _PyDict_KeysSize(PyDictKeysObject *keys); | |
| 103 | 103 | Py_ssize_t _PyDict_SizeOf(PyDictObject *); | |
| 104 | 104 | PyObject *_PyDict_Pop(PyDictObject *, PyObject *, PyObject *); | |
| 105 | + PyObject *_PyDict_Pop_KnownHash(PyDictObject *, PyObject *, Py_hash_t, PyObject *); | ||
| 105 | 106 | PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); | |
| 106 | 107 | #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) | |
| 107 | 108 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -7,6 +7,7 @@ | |||
| 7 | 7 | from random import choice | |
| 8 | 8 | import sys | |
| 9 | 9 | from test import support | |
| 10 | + import time | ||
| 10 | 11 | import unittest | |
| 11 | 12 | from weakref import proxy | |
| 12 | 13 | try: | |
@@ -1364,6 +1365,20 @@ def test(): | |||
| 1364 | 1365 | pause.reset() | |
| 1365 | 1366 | self.assertEqual(f.cache_info(), (0, (i+1)*n, m*n, i+1)) | |
| 1366 | 1367 | ||
| 1368 | + @unittest.skipUnless(threading, 'This test requires threading.') | ||
| 1369 | + def test_lru_cache_threaded3(self): | ||
| 1370 | + @self.module.lru_cache(maxsize=2) | ||
| 1371 | + def f(x): | ||
| 1372 | + time.sleep(.01) | ||
| 1373 | + return 3 * x | ||
| 1374 | + def test(i, x): | ||
| 1375 | + with self.subTest(thread=i): | ||
| 1376 | + self.assertEqual(f(x), 3 * x, i) | ||
| 1377 | + threads = [threading.Thread(target=test, args=(i, v)) | ||
| 1378 | + for i, v in enumerate([1, 2, 2, 3, 2])] | ||
| 1379 | + with support.start_threads(threads): | ||
| 1380 | + pass | ||
| 1381 | + | ||
| 1367 | 1382 | def test_need_for_rlock(self): | |
| 1368 | 1383 | # This will deadlock on an LRU cache that uses a regular lock | |
| 1369 | 1384 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -13,6 +13,10 @@ Core and Builtins | |||
| 13 | 13 | Library | |
| 14 | 14 | ------- | |
| 15 | 15 | ||
| 16 | + - Issue #28969: Fixed race condition in C implementation of functools.lru_cache. | ||
| 17 | + KeyError could be raised when cached function with full cache was | ||
| 18 | + simultaneously called from differen threads with the same uncached arguments. | ||
| 19 | + | ||
| 16 | 20 | - Issue #29142: In urllib.request, suffixes in no_proxy environment variable with | |
| 17 | 21 | leading dots could match related hostnames again (e.g. .b.c matches a.b.c). | |
| 18 | 22 | Patch by Milan Oberkirch. | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -864,42 +864,56 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds | |||
| 864 | 864 | } | |
| 865 | 865 | if (self->full && self->root.next != &self->root) { | |
| 866 | 866 | /* Use the oldest item to store the new key and result. */ | |
| 867 | - PyObject *oldkey, *oldresult; | ||
| 867 | + PyObject *oldkey, *oldresult, *popresult; | ||
| 868 | 868 | /* Extricate the oldest item. */ | |
| 869 | 869 | link = self->root.next; | |
| 870 | 870 | lru_cache_extricate_link(link); | |
| 871 | 871 | /* Remove it from the cache. | |
| 872 | 872 | The cache dict holds one reference to the link, | |
| 873 | 873 | and the linked list holds yet one reference to it. */ | |
| 874 | - if (_PyDict_DelItem_KnownHash(self->cache, link->key, | ||
| 875 | - link->hash) < 0) { | ||
| 874 | + popresult = _PyDict_Pop_KnownHash((PyDictObject *)self->cache, | ||
| 875 | + link->key, link->hash, | ||
| 876 | + Py_None); | ||
| 877 | + if (popresult == Py_None) { | ||
| 878 | + /* Getting here means that this same key was added to the | ||
| 879 | + cache while the lock was released. Since the link | ||
| 880 | + update is already done, we need only return the | ||
| 881 | + computed result and update the count of misses. */ | ||
| 882 | + Py_DECREF(popresult); | ||
| 883 | + Py_DECREF(link); | ||
| 884 | + Py_DECREF(key); | ||
| 885 | + } | ||
| 886 | + else if (popresult == NULL) { | ||
| 876 | 887 | lru_cache_append_link(self, link); | |
| 877 | 888 | Py_DECREF(key); | |
| 878 | 889 | Py_DECREF(result); | |
| 879 | 890 | return NULL; | |
| 880 | 891 | } | |
| 881 | - /* Keep a reference to the old key and old result to | ||
| 882 | - prevent their ref counts from going to zero during the | ||
| 883 | - update. That will prevent potentially arbitrary object | ||
| 884 | - clean-up code (i.e. __del__) from running while we're | ||
| 885 | - still adjusting the links. */ | ||
| 886 | - oldkey = link->key; | ||
| 887 | - oldresult = link->result; | ||
| 888 | - | ||
| 889 | - link->hash = hash; | ||
| 890 | - link->key = key; | ||
| 891 | - link->result = result; | ||
| 892 | - if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, | ||
| 893 | - hash) < 0) { | ||
| 894 | - Py_DECREF(link); | ||
| 892 | + else { | ||
| 893 | + Py_DECREF(popresult); | ||
| 894 | + /* Keep a reference to the old key and old result to | ||
| 895 | + prevent their ref counts from going to zero during the | ||
| 896 | + update. That will prevent potentially arbitrary object | ||
| 897 | + clean-up code (i.e. __del__) from running while we're | ||
| 898 | + still adjusting the links. */ | ||
| 899 | + oldkey = link->key; | ||
| 900 | + oldresult = link->result; | ||
| 901 | + | ||
| 902 | + link->hash = hash; | ||
| 903 | + link->key = key; | ||
| 904 | + link->result = result; | ||
| 905 | + if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, | ||
| 906 | + hash) < 0) { | ||
| 907 | + Py_DECREF(link); | ||
| 908 | + Py_DECREF(oldkey); | ||
| 909 | + Py_DECREF(oldresult); | ||
| 910 | + return NULL; | ||
| 911 | + } | ||
| 912 | + lru_cache_append_link(self, link); | ||
| 913 | + Py_INCREF(result); /* for return */ | ||
| 895 | 914 | Py_DECREF(oldkey); | |
| 896 | 915 | Py_DECREF(oldresult); | |
| 897 | - return NULL; | ||
| 898 | 916 | } | |
| 899 | - lru_cache_append_link(self, link); | ||
| 900 | - Py_INCREF(result); /* for return */ | ||
| 901 | - Py_DECREF(oldkey); | ||
| 902 | - Py_DECREF(oldresult); | ||
| 903 | 917 | } else { | |
| 904 | 918 | /* Put result in a new link at the front of the queue. */ | |
| 905 | 919 | link = (lru_list_elem *)PyObject_GC_New(lru_list_elem, | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1475,9 +1475,8 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, | |||
| 1475 | 1475 | ||
| 1476 | 1476 | /* Internal version of dict.pop(). */ | |
| 1477 | 1477 | PyObject * | |
| 1478 | - _PyDict_Pop(PyDictObject *mp, PyObject *key, PyObject *deflt) | ||
| 1478 | + _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *deflt) | ||
| 1479 | 1479 | { | |
| 1480 | - Py_hash_t hash; | ||
| 1481 | 1480 | PyObject *old_value, *old_key; | |
| 1482 | 1481 | PyDictKeyEntry *ep; | |
| 1483 | 1482 | PyObject **value_addr; | |
@@ -1490,12 +1489,6 @@ _PyDict_Pop(PyDictObject *mp, PyObject *key, PyObject *deflt) | |||
| 1490 | 1489 | _PyErr_SetKeyError(key); | |
| 1491 | 1490 | return NULL; | |
| 1492 | 1491 | } | |
| 1493 | - if (!PyUnicode_CheckExact(key) || | ||
| 1494 | - (hash = ((PyASCIIObject *) key)->hash) == -1) { | ||
| 1495 | - hash = PyObject_Hash(key); | ||
| 1496 | - if (hash == -1) | ||
| 1497 | - return NULL; | ||
| 1498 | - } | ||
| 1499 | 1492 | ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); | |
| 1500 | 1493 | if (ep == NULL) | |
| 1501 | 1494 | return NULL; | |
@@ -1520,6 +1513,28 @@ _PyDict_Pop(PyDictObject *mp, PyObject *key, PyObject *deflt) | |||
| 1520 | 1513 | return old_value; | |
| 1521 | 1514 | } | |
| 1522 | 1515 | ||
| 1516 | + PyObject * | ||
| 1517 | + _PyDict_Pop(PyDictObject *mp, PyObject *key, PyObject *deflt) | ||
| 1518 | + { | ||
| 1519 | + Py_hash_t hash; | ||
| 1520 | + | ||
| 1521 | + if (mp->ma_used == 0) { | ||
| 1522 | + if (deflt) { | ||
| 1523 | + Py_INCREF(deflt); | ||
| 1524 | + return deflt; | ||
| 1525 | + } | ||
| 1526 | + _PyErr_SetKeyError(key); | ||
| 1527 | + return NULL; | ||
| 1528 | + } | ||
| 1529 | + if (!PyUnicode_CheckExact(key) || | ||
| 1530 | + (hash = ((PyASCIIObject *) key)->hash) == -1) { | ||
| 1531 | + hash = PyObject_Hash(key); | ||
| 1532 | + if (hash == -1) | ||
| 1533 | + return NULL; | ||
| 1534 | + } | ||
| 1535 | + return _PyDict_Pop_KnownHash(mp, key, hash, deflt); | ||
| 1536 | + } | ||
| 1537 | + | ||
| 1523 | 1538 | /* Internal version of dict.from_keys(). It is subclass-friendly. */ | |
| 1524 | 1539 | PyObject * | |
| 1525 | 1540 | _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) | |
| Back | FazBrowse Home | New Git URL |
0 commit comments