Python code coverage for Lib/multiprocessing/queues.py

#	count	content
1	n/a	#
2	n/a	# Module implementing queues
3	n/a	#
4	n/a	# multiprocessing/queues.py
5	n/a	#
6	n/a	# Copyright (c) 2006-2008, R Oudkerk
7	n/a	# Licensed to PSF under a Contributor Agreement.
8	n/a	#
9	n/a
10	n/a	__all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
11	n/a
12	n/a	import sys
13	n/a	import os
14	n/a	import threading
15	n/a	import collections
16	n/a	import time
17	n/a	import weakref
18	n/a	import errno
19	n/a
20	n/a	from queue import Empty, Full
21	n/a
22	n/a	import _multiprocessing
23	n/a
24	n/a	from . import connection
25	n/a	from . import context
26	n/a	_ForkingPickler = context.reduction.ForkingPickler
27	n/a
28	n/a	from .util import debug, info, Finalize, register_after_fork, is_exiting
29	n/a
30	n/a	#
31	n/a	# Queue type using a pipe, buffer and thread
32	n/a	#
33	n/a
34	n/a	class Queue(object):
35	n/a
36	n/a	def __init__(self, maxsize=0, *, ctx):
37	n/a	if maxsize <= 0:
38	n/a	# Can raise ImportError (see issues #3770 and #23400)
39	n/a	from .synchronize import SEM_VALUE_MAX as maxsize
40	n/a	self._maxsize = maxsize
41	n/a	self._reader, self._writer = connection.Pipe(duplex=False)
42	n/a	self._rlock = ctx.Lock()
43	n/a	self._opid = os.getpid()
44	n/a	if sys.platform == 'win32':
45	n/a	self._wlock = None
46	n/a	else:
47	n/a	self._wlock = ctx.Lock()
48	n/a	self._sem = ctx.BoundedSemaphore(maxsize)
49	n/a	# For use by concurrent.futures
50	n/a	self._ignore_epipe = False
51	n/a
52	n/a	self._after_fork()
53	n/a
54	n/a	if sys.platform != 'win32':
55	n/a	register_after_fork(self, Queue._after_fork)
56	n/a
57	n/a	def __getstate__(self):
58	n/a	context.assert_spawning(self)
59	n/a	return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
60	n/a	self._rlock, self._wlock, self._sem, self._opid)
61	n/a
62	n/a	def __setstate__(self, state):
63	n/a	(self._ignore_epipe, self._maxsize, self._reader, self._writer,
64	n/a	self._rlock, self._wlock, self._sem, self._opid) = state
65	n/a	self._after_fork()
66	n/a
67	n/a	def _after_fork(self):
68	n/a	debug('Queue._after_fork()')
69	n/a	self._notempty = threading.Condition(threading.Lock())
70	n/a	self._buffer = collections.deque()
71	n/a	self._thread = None
72	n/a	self._jointhread = None
73	n/a	self._joincancelled = False
74	n/a	self._closed = False
75	n/a	self._close = None
76	n/a	self._send_bytes = self._writer.send_bytes
77	n/a	self._recv_bytes = self._reader.recv_bytes
78	n/a	self._poll = self._reader.poll
79	n/a
80	n/a	def put(self, obj, block=True, timeout=None):
81	n/a	assert not self._closed
82	n/a	if not self._sem.acquire(block, timeout):
83	n/a	raise Full
84	n/a
85	n/a	with self._notempty:
86	n/a	if self._thread is None:
87	n/a	self._start_thread()
88	n/a	self._buffer.append(obj)
89	n/a	self._notempty.notify()
90	n/a
91	n/a	def get(self, block=True, timeout=None):
92	n/a	if block and timeout is None:
93	n/a	with self._rlock:
94	n/a	res = self._recv_bytes()
95	n/a	self._sem.release()
96	n/a	else:
97	n/a	if block:
98	n/a	deadline = time.time() + timeout
99	n/a	if not self._rlock.acquire(block, timeout):
100	n/a	raise Empty
101	n/a	try:
102	n/a	if block:
103	n/a	timeout = deadline - time.time()
104	n/a	if timeout < 0 or not self._poll(timeout):
105	n/a	raise Empty
106	n/a	elif not self._poll():
107	n/a	raise Empty
108	n/a	res = self._recv_bytes()
109	n/a	self._sem.release()
110	n/a	finally:
111	n/a	self._rlock.release()
112	n/a	# unserialize the data after having released the lock
113	n/a	return _ForkingPickler.loads(res)
114	n/a
115	n/a	def qsize(self):
116	n/a	# Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
117	n/a	return self._maxsize - self._sem._semlock._get_value()
118	n/a
119	n/a	def empty(self):
120	n/a	return not self._poll()
121	n/a
122	n/a	def full(self):
123	n/a	return self._sem._semlock._is_zero()
124	n/a
125	n/a	def get_nowait(self):
126	n/a	return self.get(False)
127	n/a
128	n/a	def put_nowait(self, obj):
129	n/a	return self.put(obj, False)
130	n/a
131	n/a	def close(self):
132	n/a	self._closed = True
133	n/a	try:
134	n/a	self._reader.close()
135	n/a	finally:
136	n/a	close = self._close
137	n/a	if close:
138	n/a	self._close = None
139	n/a	close()
140	n/a
141	n/a	def join_thread(self):
142	n/a	debug('Queue.join_thread()')
143	n/a	assert self._closed
144	n/a	if self._jointhread:
145	n/a	self._jointhread()
146	n/a
147	n/a	def cancel_join_thread(self):
148	n/a	debug('Queue.cancel_join_thread()')
149	n/a	self._joincancelled = True
150	n/a	try:
151	n/a	self._jointhread.cancel()
152	n/a	except AttributeError:
153	n/a	pass
154	n/a
155	n/a	def _start_thread(self):
156	n/a	debug('Queue._start_thread()')
157	n/a
158	n/a	# Start thread which transfers data from buffer to pipe
159	n/a	self._buffer.clear()
160	n/a	self._thread = threading.Thread(
161	n/a	target=Queue._feed,
162	n/a	args=(self._buffer, self._notempty, self._send_bytes,
163	n/a	self._wlock, self._writer.close, self._ignore_epipe),
164	n/a	name='QueueFeederThread'
165	n/a	)
166	n/a	self._thread.daemon = True
167	n/a
168	n/a	debug('doing self._thread.start()')
169	n/a	self._thread.start()
170	n/a	debug('... done self._thread.start()')
171	n/a
172	n/a	# On process exit we will wait for data to be flushed to pipe.
173	n/a	#
174	n/a	# However, if this process created the queue then all
175	n/a	# processes which use the queue will be descendants of this
176	n/a	# process. Therefore waiting for the queue to be flushed
177	n/a	# is pointless once all the child processes have been joined.
178	n/a	created_by_this_process = (self._opid == os.getpid())
179	n/a	if not self._joincancelled and not created_by_this_process:
180	n/a	self._jointhread = Finalize(
181	n/a	self._thread, Queue._finalize_join,
182	n/a	[weakref.ref(self._thread)],
183	n/a	exitpriority=-5
184	n/a	)
185	n/a
186	n/a	# Send sentinel to the thread queue object when garbage collected
187	n/a	self._close = Finalize(
188	n/a	self, Queue._finalize_close,
189	n/a	[self._buffer, self._notempty],
190	n/a	exitpriority=10
191	n/a	)
192	n/a
193	n/a	@staticmethod
194	n/a	def _finalize_join(twr):
195	n/a	debug('joining queue thread')
196	n/a	thread = twr()
197	n/a	if thread is not None:
198	n/a	thread.join()
199	n/a	debug('... queue thread joined')
200	n/a	else:
201	n/a	debug('... queue thread already dead')
202	n/a
203	n/a	@staticmethod
204	n/a	def _finalize_close(buffer, notempty):
205	n/a	debug('telling queue thread to quit')
206	n/a	with notempty:
207	n/a	buffer.append(_sentinel)
208	n/a	notempty.notify()
209	n/a
210	n/a	@staticmethod
211	n/a	def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe):
212	n/a	debug('starting thread to feed data to pipe')
213	n/a	nacquire = notempty.acquire
214	n/a	nrelease = notempty.release
215	n/a	nwait = notempty.wait
216	n/a	bpopleft = buffer.popleft
217	n/a	sentinel = _sentinel
218	n/a	if sys.platform != 'win32':
219	n/a	wacquire = writelock.acquire
220	n/a	wrelease = writelock.release
221	n/a	else:
222	n/a	wacquire = None
223	n/a
224	n/a	try:
225	n/a	while 1:
226	n/a	nacquire()
227	n/a	try:
228	n/a	if not buffer:
229	n/a	nwait()
230	n/a	finally:
231	n/a	nrelease()
232	n/a	try:
233	n/a	while 1:
234	n/a	obj = bpopleft()
235	n/a	if obj is sentinel:
236	n/a	debug('feeder thread got sentinel -- exiting')
237	n/a	close()
238	n/a	return
239	n/a
240	n/a	# serialize the data before acquiring the lock
241	n/a	obj = _ForkingPickler.dumps(obj)
242	n/a	if wacquire is None:
243	n/a	send_bytes(obj)
244	n/a	else:
245	n/a	wacquire()
246	n/a	try:
247	n/a	send_bytes(obj)
248	n/a	finally:
249	n/a	wrelease()
250	n/a	except IndexError:
251	n/a	pass
252	n/a	except Exception as e:
253	n/a	if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE:
254	n/a	return
255	n/a	# Since this runs in a daemon thread the resources it uses
256	n/a	# may be become unusable while the process is cleaning up.
257	n/a	# We ignore errors which happen after the process has
258	n/a	# started to cleanup.
259	n/a	try:
260	n/a	if is_exiting():
261	n/a	info('error in queue thread: %s', e)
262	n/a	else:
263	n/a	import traceback
264	n/a	traceback.print_exc()
265	n/a	except Exception:
266	n/a	pass
267	n/a
268	n/a	_sentinel = object()
269	n/a
270	n/a	#
271	n/a	# A queue type which also supports join() and task_done() methods
272	n/a	#
273	n/a	# Note that if you do not call task_done() for each finished task then
274	n/a	# eventually the counter's semaphore may overflow causing Bad Things
275	n/a	# to happen.
276	n/a	#
277	n/a
278	n/a	class JoinableQueue(Queue):
279	n/a
280	n/a	def __init__(self, maxsize=0, *, ctx):
281	n/a	Queue.__init__(self, maxsize, ctx=ctx)
282	n/a	self._unfinished_tasks = ctx.Semaphore(0)
283	n/a	self._cond = ctx.Condition()
284	n/a
285	n/a	def __getstate__(self):
286	n/a	return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
287	n/a
288	n/a	def __setstate__(self, state):
289	n/a	Queue.__setstate__(self, state[:-2])
290	n/a	self._cond, self._unfinished_tasks = state[-2:]
291	n/a
292	n/a	def put(self, obj, block=True, timeout=None):
293	n/a	assert not self._closed
294	n/a	if not self._sem.acquire(block, timeout):
295	n/a	raise Full
296	n/a
297	n/a	with self._notempty, self._cond:
298	n/a	if self._thread is None:
299	n/a	self._start_thread()
300	n/a	self._buffer.append(obj)
301	n/a	self._unfinished_tasks.release()
302	n/a	self._notempty.notify()
303	n/a
304	n/a	def task_done(self):
305	n/a	with self._cond:
306	n/a	if not self._unfinished_tasks.acquire(False):
307	n/a	raise ValueError('task_done() called too many times')
308	n/a	if self._unfinished_tasks._semlock._is_zero():
309	n/a	self._cond.notify_all()
310	n/a
311	n/a	def join(self):
312	n/a	with self._cond:
313	n/a	if not self._unfinished_tasks._semlock._is_zero():
314	n/a	self._cond.wait()
315	n/a
316	n/a	#
317	n/a	# Simplified Queue type -- really just a locked pipe
318	n/a	#
319	n/a
320	n/a	class SimpleQueue(object):
321	n/a
322	n/a	def __init__(self, *, ctx):
323	n/a	self._reader, self._writer = connection.Pipe(duplex=False)
324	n/a	self._rlock = ctx.Lock()
325	n/a	self._poll = self._reader.poll
326	n/a	if sys.platform == 'win32':
327	n/a	self._wlock = None
328	n/a	else:
329	n/a	self._wlock = ctx.Lock()
330	n/a
331	n/a	def empty(self):
332	n/a	return not self._poll()
333	n/a
334	n/a	def __getstate__(self):
335	n/a	context.assert_spawning(self)
336	n/a	return (self._reader, self._writer, self._rlock, self._wlock)
337	n/a
338	n/a	def __setstate__(self, state):
339	n/a	(self._reader, self._writer, self._rlock, self._wlock) = state
340	n/a
341	n/a	def get(self):
342	n/a	with self._rlock:
343	n/a	res = self._reader.recv_bytes()
344	n/a	# unserialize the data after having released the lock
345	n/a	return _ForkingPickler.loads(res)
346	n/a
347	n/a	def put(self, obj):
348	n/a	# serialize the data before acquiring the lock
349	n/a	obj = _ForkingPickler.dumps(obj)
350	n/a	if self._wlock is None:
351	n/a	# writes to a message oriented win32 pipe are atomic
352	n/a	self._writer.send_bytes(obj)
353	n/a	else:
354	n/a	with self._wlock:
355	n/a	self._writer.send_bytes(obj)