diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 6d25469e1620c0..7d5ab67b4704cd 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -128,6 +128,8 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) @@ -402,10 +404,11 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): if set_length: util.debug('doing set_length()') set_length(i+1) + finally: + task = taskseq = job = None else: util.debug('task handler got sentinel') - try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') @@ -445,6 +448,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None while cache and thread._state != TERMINATE: try: @@ -461,6 +465,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 9b789c2a3385b2..b053342f1c26c1 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -18,6 +18,7 @@ import logging import struct import operator +import weakref import test.support import test.support.script_helper @@ -1668,6 +1669,19 @@ def sqr(x, wait=0.0): def mul(x, y): return x*y +def identity(x): + return x + +class CountedObject(object): + n_instances = 0 + + def __new__(cls): + cls.n_instances += 1 + return object.__new__(cls) + + def __del__(self): + type(self).n_instances -= 1 + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): @@ -1676,6 +1690,7 @@ def exception_throwing_generator(total, when): raise SayWhenError("Somebody said when") yield i + class _TestPool(BaseTestCase): @classmethod @@ -1910,6 +1925,19 @@ def test_wrapped_exception(self): with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + def test_release_task_refs(self): + # Issue #29861: task arguments and results should not be kept + # alive after we are done with them. + objs = [CountedObject() for i in range(10)] + refs = [weakref.ref(o) for o in objs] + self.pool.map(identity, objs) + + del objs + self.assertEqual(set(wr() for wr in refs), {None}) + # With a process pool, copies of the objects are returned, check + # they were released too. + self.assertEqual(CountedObject.n_instances, 0) + def raising(): raise KeyError("key") diff --git a/Misc/NEWS b/Misc/NEWS index e063455b986413..1afc00908c83df 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,24 +10,6 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- -- bpo-28876: ``bool(range)`` works even if ``len(range)`` - raises :exc:`OverflowError`. - -- bpo-29600: Fix wrapping coroutine return values in StopIteration. - -- Issue #29537: Restore runtime compatibility with bytecode files generated by - CPython 3.5.0 to 3.5.2, and adjust the eval loop to avoid the problems that - could be caused by the malformed variant of the BUILD_MAP_UNPACK_WITH_CALL - opcode that they may contain. Patch by Petr Viktorin, Serhiy Storchaka, - and Nick Coghlan. - -- Issue #28598: Support __rmod__ for subclasses of str being called before - str.__mod__. Patch by Martijn Pieters. - -- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for - complex subclasses and for inputs having a __complex__ method. Patch - by Serhiy Storchaka. - - bpo-29347: Fixed possibly dereferencing undefined pointers when creating weakref objects. @@ -46,43 +28,8 @@ Extension Modules Library ------- -- bpo-29884: faulthandler: Restore the old sigaltstack during teardown. - Patch by Christophe Zeitouny. - -- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. - -- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords - are not strings. Patch by Michael Seifert. - -- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. - Patch by Nikolay Kim. - -- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" - or "errors" of sys.stdin or sys.stdout are not set or are not strings. - -- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big - intables (objects that have __int__) as elements. Patch by Oren Milman. - -- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other - exception) to exception(s) raised in the dispatched methods. - Patch by Petr Motejlek. - -- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before - all pipes are closed. - -- bpo-29703: Fix asyncio to support instantiation of new event loops - in child processes. - -- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). - -- bpo-29110: Fix file object leak in aifc.open() when file is given as a - filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. - -- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter - anymore. Patch written by Jiajun Huang. - -- bpo-29532: Altering a kwarg dictionary passed to functools.partial() - no longer affects a partial object after creation. +- bpo-29861: Release references to tasks, their arguments and their results + as soon as they are finished in multiprocessing.Pool. - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, @@ -141,11 +88,6 @@ C API Documentation ------------- -- bpo-28929: Link the documentation to its source file on GitHub. - -- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to - aiosmtpd, a third-party asyncio-based replacement. - - Issue #26355: Add canonical header link on each page to corresponding major version of the documentation. Patch by Matthias Bussonnier.