Python code coverage for Lib/multiprocessing/forkserver.py

#	count	content
1	n/a	import errno
2	n/a	import os
3	n/a	import selectors
4	n/a	import signal
5	n/a	import socket
6	n/a	import struct
7	n/a	import sys
8	n/a	import threading
9	n/a
10	n/a	from . import connection
11	n/a	from . import process
12	n/a	from .context import reduction
13	n/a	from . import semaphore_tracker
14	n/a	from . import spawn
15	n/a	from . import util
16	n/a
17	n/a	__all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process',
18	n/a	'set_forkserver_preload']
19	n/a
20	n/a	#
21	n/a	#
22	n/a	#
23	n/a
24	n/a	MAXFDS_TO_SEND = 256
25	n/a	UNSIGNED_STRUCT = struct.Struct('Q') # large enough for pid_t
26	n/a
27	n/a	#
28	n/a	# Forkserver class
29	n/a	#
30	n/a
31	n/a	class ForkServer(object):
32	n/a
33	n/a	def __init__(self):
34	n/a	self._forkserver_address = None
35	n/a	self._forkserver_alive_fd = None
36	n/a	self._inherited_fds = None
37	n/a	self._lock = threading.Lock()
38	n/a	self._preload_modules = ['__main__']
39	n/a
40	n/a	def set_forkserver_preload(self, modules_names):
41	n/a	'''Set list of module names to try to load in forkserver process.'''
42	n/a	if not all(type(mod) is str for mod in self._preload_modules):
43	n/a	raise TypeError('module_names must be a list of strings')
44	n/a	self._preload_modules = modules_names
45	n/a
46	n/a	def get_inherited_fds(self):
47	n/a	'''Return list of fds inherited from parent process.
48	n/a
49	n/a	This returns None if the current process was not started by fork
50	n/a	server.
51	n/a	'''
52	n/a	return self._inherited_fds
53	n/a
54	n/a	def connect_to_new_process(self, fds):
55	n/a	'''Request forkserver to create a child process.
56	n/a
57	n/a	Returns a pair of fds (status_r, data_w). The calling process can read
58	n/a	the child process's pid and (eventually) its returncode from status_r.
59	n/a	The calling process should write to data_w the pickled preparation and
60	n/a	process data.
61	n/a	'''
62	n/a	self.ensure_running()
63	n/a	if len(fds) + 4 >= MAXFDS_TO_SEND:
64	n/a	raise ValueError('too many fds')
65	n/a	with socket.socket(socket.AF_UNIX) as client:
66	n/a	client.connect(self._forkserver_address)
67	n/a	parent_r, child_w = os.pipe()
68	n/a	child_r, parent_w = os.pipe()
69	n/a	allfds = [child_r, child_w, self._forkserver_alive_fd,
70	n/a	semaphore_tracker.getfd()]
71	n/a	allfds += fds
72	n/a	try:
73	n/a	reduction.sendfds(client, allfds)
74	n/a	return parent_r, parent_w
75	n/a	except:
76	n/a	os.close(parent_r)
77	n/a	os.close(parent_w)
78	n/a	raise
79	n/a	finally:
80	n/a	os.close(child_r)
81	n/a	os.close(child_w)
82	n/a
83	n/a	def ensure_running(self):
84	n/a	'''Make sure that a fork server is running.
85	n/a
86	n/a	This can be called from any process. Note that usually a child
87	n/a	process will just reuse the forkserver started by its parent, so
88	n/a	ensure_running() will do nothing.
89	n/a	'''
90	n/a	with self._lock:
91	n/a	semaphore_tracker.ensure_running()
92	n/a	if self._forkserver_alive_fd is not None:
93	n/a	return
94	n/a
95	n/a	cmd = ('from multiprocessing.forkserver import main; ' +
96	n/a	'main(%d, %d, %r, **%r)')
97	n/a
98	n/a	if self._preload_modules:
99	n/a	desired_keys = {'main_path', 'sys_path'}
100	n/a	data = spawn.get_preparation_data('ignore')
101	n/a	data = dict((x,y) for (x,y) in data.items()
102	n/a	if x in desired_keys)
103	n/a	else:
104	n/a	data = {}
105	n/a
106	n/a	with socket.socket(socket.AF_UNIX) as listener:
107	n/a	address = connection.arbitrary_address('AF_UNIX')
108	n/a	listener.bind(address)
109	n/a	os.chmod(address, 0o600)
110	n/a	listener.listen()
111	n/a
112	n/a	# all client processes own the write end of the "alive" pipe;
113	n/a	# when they all terminate the read end becomes ready.
114	n/a	alive_r, alive_w = os.pipe()
115	n/a	try:
116	n/a	fds_to_pass = [listener.fileno(), alive_r]
117	n/a	cmd %= (listener.fileno(), alive_r, self._preload_modules,
118	n/a	data)
119	n/a	exe = spawn.get_executable()
120	n/a	args = [exe] + util._args_from_interpreter_flags()
121	n/a	args += ['-c', cmd]
122	n/a	pid = util.spawnv_passfds(exe, args, fds_to_pass)
123	n/a	except:
124	n/a	os.close(alive_w)
125	n/a	raise
126	n/a	finally:
127	n/a	os.close(alive_r)
128	n/a	self._forkserver_address = address
129	n/a	self._forkserver_alive_fd = alive_w
130	n/a
131	n/a	#
132	n/a	#
133	n/a	#
134	n/a
135	n/a	def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
136	n/a	'''Run forkserver.'''
137	n/a	if preload:
138	n/a	if '__main__' in preload and main_path is not None:
139	n/a	process.current_process()._inheriting = True
140	n/a	try:
141	n/a	spawn.import_main_path(main_path)
142	n/a	finally:
143	n/a	del process.current_process()._inheriting
144	n/a	for modname in preload:
145	n/a	try:
146	n/a	__import__(modname)
147	n/a	except ImportError:
148	n/a	pass
149	n/a
150	n/a	util._close_stdin()
151	n/a
152	n/a	# ignoring SIGCHLD means no need to reap zombie processes
153	n/a	handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN)
154	n/a	with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \
155	n/a	selectors.DefaultSelector() as selector:
156	n/a	_forkserver._forkserver_address = listener.getsockname()
157	n/a
158	n/a	selector.register(listener, selectors.EVENT_READ)
159	n/a	selector.register(alive_r, selectors.EVENT_READ)
160	n/a
161	n/a	while True:
162	n/a	try:
163	n/a	while True:
164	n/a	rfds = [key.fileobj for (key, events) in selector.select()]
165	n/a	if rfds:
166	n/a	break
167	n/a
168	n/a	if alive_r in rfds:
169	n/a	# EOF because no more client processes left
170	n/a	assert os.read(alive_r, 1) == b''
171	n/a	raise SystemExit
172	n/a
173	n/a	assert listener in rfds
174	n/a	with listener.accept()[0] as s:
175	n/a	code = 1
176	n/a	if os.fork() == 0:
177	n/a	try:
178	n/a	_serve_one(s, listener, alive_r, handler)
179	n/a	except Exception:
180	n/a	sys.excepthook(*sys.exc_info())
181	n/a	sys.stderr.flush()
182	n/a	finally:
183	n/a	os._exit(code)
184	n/a
185	n/a	except OSError as e:
186	n/a	if e.errno != errno.ECONNABORTED:
187	n/a	raise
188	n/a
189	n/a	def _serve_one(s, listener, alive_r, handler):
190	n/a	# close unnecessary stuff and reset SIGCHLD handler
191	n/a	listener.close()
192	n/a	os.close(alive_r)
193	n/a	signal.signal(signal.SIGCHLD, handler)
194	n/a
195	n/a	# receive fds from parent process
196	n/a	fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
197	n/a	s.close()
198	n/a	assert len(fds) <= MAXFDS_TO_SEND
199	n/a	(child_r, child_w, _forkserver._forkserver_alive_fd,
200	n/a	stfd, *_forkserver._inherited_fds) = fds
201	n/a	semaphore_tracker._semaphore_tracker._fd = stfd
202	n/a
203	n/a	# send pid to client processes
204	n/a	write_unsigned(child_w, os.getpid())
205	n/a
206	n/a	# reseed random number generator
207	n/a	if 'random' in sys.modules:
208	n/a	import random
209	n/a	random.seed()
210	n/a
211	n/a	# run process object received over pipe
212	n/a	code = spawn._main(child_r)
213	n/a
214	n/a	# write the exit code to the pipe
215	n/a	write_unsigned(child_w, code)
216	n/a
217	n/a	#
218	n/a	# Read and write unsigned numbers
219	n/a	#
220	n/a
221	n/a	def read_unsigned(fd):
222	n/a	data = b''
223	n/a	length = UNSIGNED_STRUCT.size
224	n/a	while len(data) < length:
225	n/a	s = os.read(fd, length - len(data))
226	n/a	if not s:
227	n/a	raise EOFError('unexpected EOF')
228	n/a	data += s
229	n/a	return UNSIGNED_STRUCT.unpack(data)[0]
230	n/a
231	n/a	def write_unsigned(fd, n):
232	n/a	msg = UNSIGNED_STRUCT.pack(n)
233	n/a	while msg:
234	n/a	nbytes = os.write(fd, msg)
235	n/a	if nbytes == 0:
236	n/a	raise RuntimeError('should not get here')
237	n/a	msg = msg[nbytes:]
238	n/a
239	n/a	#
240	n/a	#
241	n/a	#
242	n/a
243	n/a	_forkserver = ForkServer()
244	n/a	ensure_running = _forkserver.ensure_running
245	n/a	get_inherited_fds = _forkserver.get_inherited_fds
246	n/a	connect_to_new_process = _forkserver.connect_to_new_process
247	n/a	set_forkserver_preload = _forkserver.set_forkserver_preload