Python code coverage for Lib/multiprocessing/forking.py

#	count	content
1	n/a	#
2	n/a	# Module for starting a process object using os.fork() or CreateProcess()
3	n/a	#
4	n/a	# multiprocessing/forking.py
5	n/a	#
6	n/a	# Copyright (c) 2006-2008, R Oudkerk
7	n/a	# Licensed to PSF under a Contributor Agreement.
8	n/a	#
9	n/a
10	n/a	import io
11	n/a	import os
12	n/a	import pickle
13	n/a	import sys
14	n/a	import signal
15	n/a	import errno
16	n/a
17	n/a	from multiprocessing import util, process
18	n/a
19	n/a	__all__ = ['Popen', 'assert_spawning', 'duplicate', 'close', 'ForkingPickler']
20	n/a
21	n/a	#
22	n/a	# Check that the current thread is spawning a child process
23	n/a	#
24	n/a
25	n/a	def assert_spawning(self):
26	n/a	if not Popen.thread_is_spawning():
27	n/a	raise RuntimeError(
28	n/a	'%s objects should only be shared between processes'
29	n/a	' through inheritance' % type(self).__name__
30	n/a	)
31	n/a
32	n/a	#
33	n/a	# Try making some callable types picklable
34	n/a	#
35	n/a
36	n/a	from pickle import Pickler
37	n/a	from copyreg import dispatch_table
38	n/a
39	n/a	class ForkingPickler(Pickler):
40	n/a	_extra_reducers = {}
41	n/a	def __init__(self, *args):
42	n/a	Pickler.__init__(self, *args)
43	n/a	self.dispatch_table = dispatch_table.copy()
44	n/a	self.dispatch_table.update(self._extra_reducers)
45	n/a	@classmethod
46	n/a	def register(cls, type, reduce):
47	n/a	cls._extra_reducers[type] = reduce
48	n/a
49	n/a	@staticmethod
50	n/a	def dumps(obj):
51	n/a	buf = io.BytesIO()
52	n/a	ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)
53	n/a	return buf.getbuffer()
54	n/a
55	n/a	loads = pickle.loads
56	n/a
57	n/a
58	n/a	def _reduce_method(m):
59	n/a	if m.__self__ is None:
60	n/a	return getattr, (m.__class__, m.__func__.__name__)
61	n/a	else:
62	n/a	return getattr, (m.__self__, m.__func__.__name__)
63	n/a	class _C:
64	n/a	def f(self):
65	n/a	pass
66	n/a	ForkingPickler.register(type(_C().f), _reduce_method)
67	n/a
68	n/a
69	n/a	def _reduce_method_descriptor(m):
70	n/a	return getattr, (m.__objclass__, m.__name__)
71	n/a	ForkingPickler.register(type(list.append), _reduce_method_descriptor)
72	n/a	ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
73	n/a
74	n/a	try:
75	n/a	from functools import partial
76	n/a	except ImportError:
77	n/a	pass
78	n/a	else:
79	n/a	def _reduce_partial(p):
80	n/a	return _rebuild_partial, (p.func, p.args, p.keywords or {})
81	n/a	def _rebuild_partial(func, args, keywords):
82	n/a	return partial(func, args, *keywords)
83	n/a	ForkingPickler.register(partial, _reduce_partial)
84	n/a
85	n/a	#
86	n/a	# Unix
87	n/a	#
88	n/a
89	n/a	if sys.platform != 'win32':
90	n/a	duplicate = os.dup
91	n/a	close = os.close
92	n/a
93	n/a	#
94	n/a	# We define a Popen class similar to the one from subprocess, but
95	n/a	# whose constructor takes a process object as its argument.
96	n/a	#
97	n/a
98	n/a	class Popen(object):
99	n/a
100	n/a	def __init__(self, process_obj):
101	n/a	sys.stdout.flush()
102	n/a	sys.stderr.flush()
103	n/a	self.returncode = None
104	n/a
105	n/a	r, w = os.pipe()
106	n/a	self.sentinel = r
107	n/a
108	n/a	self.pid = os.fork()
109	n/a	if self.pid == 0:
110	n/a	os.close(r)
111	n/a	if 'random' in sys.modules:
112	n/a	import random
113	n/a	random.seed()
114	n/a	code = process_obj._bootstrap()
115	n/a	os._exit(code)
116	n/a
117	n/a	# `w` will be closed when the child exits, at which point `r`
118	n/a	# will become ready for reading (using e.g. select()).
119	n/a	os.close(w)
120	n/a	util.Finalize(self, os.close, (r,))
121	n/a
122	n/a	def poll(self, flag=os.WNOHANG):
123	n/a	if self.returncode is None:
124	n/a	while True:
125	n/a	try:
126	n/a	pid, sts = os.waitpid(self.pid, flag)
127	n/a	except OSError as e:
128	n/a	if e.errno == errno.EINTR:
129	n/a	continue
130	n/a	# Child process not yet created. See #1731717
131	n/a	# e.errno == errno.ECHILD == 10
132	n/a	return None
133	n/a	else:
134	n/a	break
135	n/a	if pid == self.pid:
136	n/a	if os.WIFSIGNALED(sts):
137	n/a	self.returncode = -os.WTERMSIG(sts)
138	n/a	else:
139	n/a	assert os.WIFEXITED(sts)
140	n/a	self.returncode = os.WEXITSTATUS(sts)
141	n/a	return self.returncode
142	n/a
143	n/a	def wait(self, timeout=None):
144	n/a	if self.returncode is None:
145	n/a	if timeout is not None:
146	n/a	from .connection import wait
147	n/a	if not wait([self.sentinel], timeout):
148	n/a	return None
149	n/a	# This shouldn't block if wait() returned successfully.
150	n/a	return self.poll(os.WNOHANG if timeout == 0.0 else 0)
151	n/a	return self.returncode
152	n/a
153	n/a	def terminate(self):
154	n/a	if self.returncode is None:
155	n/a	try:
156	n/a	os.kill(self.pid, signal.SIGTERM)
157	n/a	except OSError:
158	n/a	if self.wait(timeout=0.1) is None:
159	n/a	raise
160	n/a
161	n/a	@staticmethod
162	n/a	def thread_is_spawning():
163	n/a	return False
164	n/a
165	n/a	#
166	n/a	# Windows
167	n/a	#
168	n/a
169	n/a	else:
170	n/a	import _thread
171	n/a	import msvcrt
172	n/a	import _winapi
173	n/a
174	n/a	from pickle import load, HIGHEST_PROTOCOL
175	n/a
176	n/a	def dump(obj, file, protocol=None):
177	n/a	ForkingPickler(file, protocol).dump(obj)
178	n/a
179	n/a	#
180	n/a	#
181	n/a	#
182	n/a
183	n/a	TERMINATE = 0x10000
184	n/a	WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
185	n/a	WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
186	n/a
187	n/a	close = _winapi.CloseHandle
188	n/a
189	n/a	#
190	n/a	# _python_exe is the assumed path to the python executable.
191	n/a	# People embedding Python want to modify it.
192	n/a	#
193	n/a
194	n/a	if WINSERVICE:
195	n/a	_python_exe = os.path.join(sys.exec_prefix, 'python.exe')
196	n/a	else:
197	n/a	_python_exe = sys.executable
198	n/a
199	n/a	def set_executable(exe):
200	n/a	global _python_exe
201	n/a	_python_exe = exe
202	n/a
203	n/a	#
204	n/a	#
205	n/a	#
206	n/a
207	n/a	def duplicate(handle, target_process=None, inheritable=False):
208	n/a	if target_process is None:
209	n/a	target_process = _winapi.GetCurrentProcess()
210	n/a	return _winapi.DuplicateHandle(
211	n/a	_winapi.GetCurrentProcess(), handle, target_process,
212	n/a	0, inheritable, _winapi.DUPLICATE_SAME_ACCESS
213	n/a	)
214	n/a
215	n/a	#
216	n/a	# We define a Popen class similar to the one from subprocess, but
217	n/a	# whose constructor takes a process object as its argument.
218	n/a	#
219	n/a
220	n/a	class Popen(object):
221	n/a	'''
222	n/a	Start a subprocess to run the code of a process object
223	n/a	'''
224	n/a	_tls = _thread._local()
225	n/a
226	n/a	def __init__(self, process_obj):
227	n/a	cmd = ' '.join('"%s"' % x for x in get_command_line())
228	n/a	prep_data = get_preparation_data(process_obj._name)
229	n/a
230	n/a	# create pipe for communication with child
231	n/a	rfd, wfd = os.pipe()
232	n/a
233	n/a	# get handle for read end of the pipe and make it inheritable
234	n/a	rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
235	n/a	os.close(rfd)
236	n/a
237	n/a	with open(wfd, 'wb', closefd=True) as to_child:
238	n/a	# start process
239	n/a	try:
240	n/a	hp, ht, pid, tid = _winapi.CreateProcess(
241	n/a	_python_exe, cmd + (' %s' % rhandle),
242	n/a	None, None, 1, 0, None, None, None
243	n/a	)
244	n/a	_winapi.CloseHandle(ht)
245	n/a	finally:
246	n/a	close(rhandle)
247	n/a
248	n/a	# set attributes of self
249	n/a	self.pid = pid
250	n/a	self.returncode = None
251	n/a	self._handle = hp
252	n/a	self.sentinel = int(hp)
253	n/a	util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
254	n/a
255	n/a	# send information to child
256	n/a	Popen._tls.process_handle = int(hp)
257	n/a	try:
258	n/a	dump(prep_data, to_child, HIGHEST_PROTOCOL)
259	n/a	dump(process_obj, to_child, HIGHEST_PROTOCOL)
260	n/a	finally:
261	n/a	del Popen._tls.process_handle
262	n/a
263	n/a	@staticmethod
264	n/a	def thread_is_spawning():
265	n/a	return getattr(Popen._tls, 'process_handle', None) is not None
266	n/a
267	n/a	@staticmethod
268	n/a	def duplicate_for_child(handle):
269	n/a	return duplicate(handle, Popen._tls.process_handle)
270	n/a
271	n/a	def wait(self, timeout=None):
272	n/a	if self.returncode is None:
273	n/a	if timeout is None:
274	n/a	msecs = _winapi.INFINITE
275	n/a	else:
276	n/a	msecs = max(0, int(timeout * 1000 + 0.5))
277	n/a
278	n/a	res = _winapi.WaitForSingleObject(int(self._handle), msecs)
279	n/a	if res == _winapi.WAIT_OBJECT_0:
280	n/a	code = _winapi.GetExitCodeProcess(self._handle)
281	n/a	if code == TERMINATE:
282	n/a	code = -signal.SIGTERM
283	n/a	self.returncode = code
284	n/a
285	n/a	return self.returncode
286	n/a
287	n/a	def poll(self):
288	n/a	return self.wait(timeout=0)
289	n/a
290	n/a	def terminate(self):
291	n/a	if self.returncode is None:
292	n/a	try:
293	n/a	_winapi.TerminateProcess(int(self._handle), TERMINATE)
294	n/a	except OSError:
295	n/a	if self.wait(timeout=1.0) is None:
296	n/a	raise
297	n/a
298	n/a	#
299	n/a	#
300	n/a	#
301	n/a
302	n/a	def is_forking(argv):
303	n/a	'''
304	n/a	Return whether commandline indicates we are forking
305	n/a	'''
306	n/a	if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
307	n/a	assert len(argv) == 3
308	n/a	return True
309	n/a	else:
310	n/a	return False
311	n/a
312	n/a
313	n/a	def freeze_support():
314	n/a	'''
315	n/a	Run code for process object if this in not the main process
316	n/a	'''
317	n/a	if is_forking(sys.argv):
318	n/a	main()
319	n/a	sys.exit()
320	n/a
321	n/a
322	n/a	def get_command_line():
323	n/a	'''
324	n/a	Returns prefix of command line used for spawning a child process
325	n/a	'''
326	n/a	if getattr(process.current_process(), '_inheriting', False):
327	n/a	raise RuntimeError('''
328	n/a	Attempt to start a new process before the current process
329	n/a	has finished its bootstrapping phase.
330	n/a
331	n/a	This probably means that you are on Windows and you have
332	n/a	forgotten to use the proper idiom in the main module:
333	n/a
334	n/a	if __name__ == '__main__':
335	n/a	freeze_support()
336	n/a	...
337	n/a
338	n/a	The "freeze_support()" line can be omitted if the program
339	n/a	is not going to be frozen to produce a Windows executable.''')
340	n/a
341	n/a	if getattr(sys, 'frozen', False):
342	n/a	return [sys.executable, '--multiprocessing-fork']
343	n/a	else:
344	n/a	prog = 'from multiprocessing.forking import main; main()'
345	n/a	opts = util._args_from_interpreter_flags()
346	n/a	return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
347	n/a
348	n/a
349	n/a	def main():
350	n/a	'''
351	n/a	Run code specifed by data received over pipe
352	n/a	'''
353	n/a	assert is_forking(sys.argv)
354	n/a
355	n/a	handle = int(sys.argv[-1])
356	n/a	fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
357	n/a	from_parent = os.fdopen(fd, 'rb')
358	n/a
359	n/a	process.current_process()._inheriting = True
360	n/a	preparation_data = load(from_parent)
361	n/a	prepare(preparation_data)
362	n/a	self = load(from_parent)
363	n/a	process.current_process()._inheriting = False
364	n/a
365	n/a	from_parent.close()
366	n/a
367	n/a	exitcode = self._bootstrap()
368	n/a	sys.exit(exitcode)
369	n/a
370	n/a
371	n/a	def get_preparation_data(name):
372	n/a	'''
373	n/a	Return info about parent needed by child to unpickle process object
374	n/a	'''
375	n/a	from .util import _logger, _log_to_stderr
376	n/a
377	n/a	d = dict(
378	n/a	name=name,
379	n/a	sys_path=sys.path,
380	n/a	sys_argv=sys.argv,
381	n/a	log_to_stderr=_log_to_stderr,
382	n/a	orig_dir=process.ORIGINAL_DIR,
383	n/a	authkey=process.current_process().authkey,
384	n/a	)
385	n/a
386	n/a	if _logger is not None:
387	n/a	d['log_level'] = _logger.getEffectiveLevel()
388	n/a
389	n/a	if not WINEXE and not WINSERVICE:
390	n/a	main_path = getattr(sys.modules['__main__'], '__file__', None)
391	n/a	if not main_path and sys.argv[0] not in ('', '-c'):
392	n/a	main_path = sys.argv[0]
393	n/a	if main_path is not None:
394	n/a	if not os.path.isabs(main_path) and \
395	n/a	process.ORIGINAL_DIR is not None:
396	n/a	main_path = os.path.join(process.ORIGINAL_DIR, main_path)
397	n/a	d['main_path'] = os.path.normpath(main_path)
398	n/a
399	n/a	return d
400	n/a
401	n/a	#
402	n/a	# Prepare current process
403	n/a	#
404	n/a
405	n/a	old_main_modules = []
406	n/a
407	n/a	def prepare(data):
408	n/a	'''
409	n/a	Try to get current process ready to unpickle process object
410	n/a	'''
411	n/a	old_main_modules.append(sys.modules['__main__'])
412	n/a
413	n/a	if 'name' in data:
414	n/a	process.current_process().name = data['name']
415	n/a
416	n/a	if 'authkey' in data:
417	n/a	process.current_process()._authkey = data['authkey']
418	n/a
419	n/a	if 'log_to_stderr' in data and data['log_to_stderr']:
420	n/a	util.log_to_stderr()
421	n/a
422	n/a	if 'log_level' in data:
423	n/a	util.get_logger().setLevel(data['log_level'])
424	n/a
425	n/a	if 'sys_path' in data:
426	n/a	sys.path = data['sys_path']
427	n/a
428	n/a	if 'sys_argv' in data:
429	n/a	sys.argv = data['sys_argv']
430	n/a
431	n/a	if 'dir' in data:
432	n/a	os.chdir(data['dir'])
433	n/a
434	n/a	if 'orig_dir' in data:
435	n/a	process.ORIGINAL_DIR = data['orig_dir']
436	n/a
437	n/a	if 'main_path' in data:
438	n/a	# XXX (ncoghlan): The following code makes several bogus
439	n/a	# assumptions regarding the relationship between __file__
440	n/a	# and a module's real name. See PEP 302 and issue #10845
441	n/a	main_path = data['main_path']
442	n/a	main_name = os.path.splitext(os.path.basename(main_path))[0]
443	n/a	if main_name == '__init__':
444	n/a	main_name = os.path.basename(os.path.dirname(main_path))
445	n/a
446	n/a	if main_name == '__main__':
447	n/a	main_module = sys.modules['__main__']
448	n/a	main_module.__file__ = main_path
449	n/a	elif main_name != 'ipython':
450	n/a	# Main modules not actually called __main__.py may
451	n/a	# contain additional code that should still be executed
452	n/a	import importlib
453	n/a	import types
454	n/a
455	n/a	if main_path is None:
456	n/a	dirs = None
457	n/a	elif os.path.basename(main_path).startswith('__init__.py'):
458	n/a	dirs = [os.path.dirname(os.path.dirname(main_path))]
459	n/a	else:
460	n/a	dirs = [os.path.dirname(main_path)]
461	n/a
462	n/a	assert main_name not in sys.modules, main_name
463	n/a	sys.modules.pop('__mp_main__', None)
464	n/a	# We should not try to load __main__
465	n/a	# since that would execute 'if __name__ == "__main__"'
466	n/a	# clauses, potentially causing a psuedo fork bomb.
467	n/a	loader = importlib.find_loader(main_name, path=dirs)
468	n/a	main_module = types.ModuleType(main_name)
469	n/a	try:
470	n/a	loader.init_module_attrs(main_module)
471	n/a	except AttributeError: # init_module_attrs is optional
472	n/a	pass
473	n/a	main_module.__name__ = '__mp_main__'
474	n/a	code = loader.get_code(main_name)
475	n/a	exec(code, main_module.__dict__)
476	n/a
477	n/a	sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module