Python code coverage for Lib/compileall.py

#	count	content
1	n/a	"""Module/script to byte-compile all .py files to .pyc files.
2	n/a
3	n/a	When called as a script with arguments, this compiles the directories
4	n/a	given as arguments recursively; the -l option prevents it from
5	n/a	recursing into directories.
6	n/a
7	n/a	Without arguments, if compiles all modules on sys.path, without
8	n/a	recursing into subdirectories. (Even though it should do so for
9	n/a	packages -- for now, you'll have to deal with packages separately.)
10	n/a
11	n/a	See module py_compile for details of the actual byte-compilation.
12	n/a	"""
13	n/a	import os
14	n/a	import sys
15	n/a	import importlib.util
16	n/a	import py_compile
17	n/a	import struct
18	n/a
19	n/a	try:
20	n/a	from concurrent.futures import ProcessPoolExecutor
21	n/a	except ImportError:
22	n/a	ProcessPoolExecutor = None
23	n/a	from functools import partial
24	n/a
25	n/a	__all__ = ["compile_dir","compile_file","compile_path"]
26	n/a
27	n/a	def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
28	n/a	if quiet < 2 and isinstance(dir, os.PathLike):
29	n/a	dir = os.fspath(dir)
30	n/a	if not quiet:
31	n/a	print('Listing {!r}...'.format(dir))
32	n/a	try:
33	n/a	names = os.listdir(dir)
34	n/a	except OSError:
35	n/a	if quiet < 2:
36	n/a	print("Can't list {!r}".format(dir))
37	n/a	names = []
38	n/a	names.sort()
39	n/a	for name in names:
40	n/a	if name == '__pycache__':
41	n/a	continue
42	n/a	fullname = os.path.join(dir, name)
43	n/a	if ddir is not None:
44	n/a	dfile = os.path.join(ddir, name)
45	n/a	else:
46	n/a	dfile = None
47	n/a	if not os.path.isdir(fullname):
48	n/a	yield fullname
49	n/a	elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
50	n/a	os.path.isdir(fullname) and not os.path.islink(fullname)):
51	n/a	yield from _walk_dir(fullname, ddir=dfile,
52	n/a	maxlevels=maxlevels - 1, quiet=quiet)
53	n/a
54	n/a	def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
55	n/a	quiet=0, legacy=False, optimize=-1, workers=1):
56	n/a	"""Byte-compile all modules in the given directory tree.
57	n/a
58	n/a	Arguments (only dir is required):
59	n/a
60	n/a	dir: the directory to byte-compile
61	n/a	maxlevels: maximum recursion level (default 10)
62	n/a	ddir: the directory that will be prepended to the path to the
63	n/a	file as it is compiled into each byte-code file.
64	n/a	force: if True, force compilation, even if timestamps are up-to-date
65	n/a	quiet: full output with False or 0, errors only with 1,
66	n/a	no output with 2
67	n/a	legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
68	n/a	optimize: optimization level or -1 for level of the interpreter
69	n/a	workers: maximum number of parallel workers
70	n/a	"""
71	n/a	if workers is not None and workers < 0:
72	n/a	raise ValueError('workers must be greater or equal to 0')
73	n/a
74	n/a	files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
75	n/a	ddir=ddir)
76	n/a	success = True
77	n/a	if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
78	n/a	workers = workers or None
79	n/a	with ProcessPoolExecutor(max_workers=workers) as executor:
80	n/a	results = executor.map(partial(compile_file,
81	n/a	ddir=ddir, force=force,
82	n/a	rx=rx, quiet=quiet,
83	n/a	legacy=legacy,
84	n/a	optimize=optimize),
85	n/a	files)
86	n/a	success = min(results, default=True)
87	n/a	else:
88	n/a	for file in files:
89	n/a	if not compile_file(file, ddir, force, rx, quiet,
90	n/a	legacy, optimize):
91	n/a	success = False
92	n/a	return success
93	n/a
94	n/a	def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
95	n/a	legacy=False, optimize=-1):
96	n/a	"""Byte-compile one file.
97	n/a
98	n/a	Arguments (only fullname is required):
99	n/a
100	n/a	fullname: the file to byte-compile
101	n/a	ddir: if given, the directory name compiled in to the
102	n/a	byte-code file.
103	n/a	force: if True, force compilation, even if timestamps are up-to-date
104	n/a	quiet: full output with False or 0, errors only with 1,
105	n/a	no output with 2
106	n/a	legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
107	n/a	optimize: optimization level or -1 for level of the interpreter
108	n/a	"""
109	n/a	success = True
110	n/a	if quiet < 2 and isinstance(fullname, os.PathLike):
111	n/a	fullname = os.fspath(fullname)
112	n/a	name = os.path.basename(fullname)
113	n/a	if ddir is not None:
114	n/a	dfile = os.path.join(ddir, name)
115	n/a	else:
116	n/a	dfile = None
117	n/a	if rx is not None:
118	n/a	mo = rx.search(fullname)
119	n/a	if mo:
120	n/a	return success
121	n/a	if os.path.isfile(fullname):
122	n/a	if legacy:
123	n/a	cfile = fullname + 'c'
124	n/a	else:
125	n/a	if optimize >= 0:
126	n/a	opt = optimize if optimize >= 1 else ''
127	n/a	cfile = importlib.util.cache_from_source(
128	n/a	fullname, optimization=opt)
129	n/a	else:
130	n/a	cfile = importlib.util.cache_from_source(fullname)
131	n/a	cache_dir = os.path.dirname(cfile)
132	n/a	head, tail = name[:-3], name[-3:]
133	n/a	if tail == '.py':
134	n/a	if not force:
135	n/a	try:
136	n/a	mtime = int(os.stat(fullname).st_mtime)
137	n/a	expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
138	n/a	mtime)
139	n/a	with open(cfile, 'rb') as chandle:
140	n/a	actual = chandle.read(8)
141	n/a	if expect == actual:
142	n/a	return success
143	n/a	except OSError:
144	n/a	pass
145	n/a	if not quiet:
146	n/a	print('Compiling {!r}...'.format(fullname))
147	n/a	try:
148	n/a	ok = py_compile.compile(fullname, cfile, dfile, True,
149	n/a	optimize=optimize)
150	n/a	except py_compile.PyCompileError as err:
151	n/a	success = False
152	n/a	if quiet >= 2:
153	n/a	return success
154	n/a	elif quiet:
155	n/a	print('*** Error compiling {!r}...'.format(fullname))
156	n/a	else:
157	n/a	print('*** ', end='')
158	n/a	# escape non-printable characters in msg
159	n/a	msg = err.msg.encode(sys.stdout.encoding,
160	n/a	errors='backslashreplace')
161	n/a	msg = msg.decode(sys.stdout.encoding)
162	n/a	print(msg)
163	n/a	except (SyntaxError, UnicodeError, OSError) as e:
164	n/a	success = False
165	n/a	if quiet >= 2:
166	n/a	return success
167	n/a	elif quiet:
168	n/a	print('*** Error compiling {!r}...'.format(fullname))
169	n/a	else:
170	n/a	print('*** ', end='')
171	n/a	print(e.__class__.__name__ + ':', e)
172	n/a	else:
173	n/a	if ok == 0:
174	n/a	success = False
175	n/a	return success
176	n/a
177	n/a	def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
178	n/a	legacy=False, optimize=-1):
179	n/a	"""Byte-compile all module on sys.path.
180	n/a
181	n/a	Arguments (all optional):
182	n/a
183	n/a	skip_curdir: if true, skip current directory (default True)
184	n/a	maxlevels: max recursion level (default 0)
185	n/a	force: as for compile_dir() (default False)
186	n/a	quiet: as for compile_dir() (default 0)
187	n/a	legacy: as for compile_dir() (default False)
188	n/a	optimize: as for compile_dir() (default -1)
189	n/a	"""
190	n/a	success = True
191	n/a	for dir in sys.path:
192	n/a	if (not dir or dir == os.curdir) and skip_curdir:
193	n/a	if quiet < 2:
194	n/a	print('Skipping current directory')
195	n/a	else:
196	n/a	success = success and compile_dir(dir, maxlevels, None,
197	n/a	force, quiet=quiet,
198	n/a	legacy=legacy, optimize=optimize)
199	n/a	return success
200	n/a
201	n/a
202	n/a	def main():
203	n/a	"""Script main program."""
204	n/a	import argparse
205	n/a
206	n/a	parser = argparse.ArgumentParser(
207	n/a	description='Utilities to support installing Python libraries.')
208	n/a	parser.add_argument('-l', action='store_const', const=0,
209	n/a	default=10, dest='maxlevels',
210	n/a	help="don't recurse into subdirectories")
211	n/a	parser.add_argument('-r', type=int, dest='recursion',
212	n/a	help=('control the maximum recursion level. '
213	n/a	'if `-l` and `-r` options are specified, '
214	n/a	'then `-r` takes precedence.'))
215	n/a	parser.add_argument('-f', action='store_true', dest='force',
216	n/a	help='force rebuild even if timestamps are up to date')
217	n/a	parser.add_argument('-q', action='count', dest='quiet', default=0,
218	n/a	help='output only error messages; -qq will suppress '
219	n/a	'the error messages as well.')
220	n/a	parser.add_argument('-b', action='store_true', dest='legacy',
221	n/a	help='use legacy (pre-PEP3147) compiled file locations')
222	n/a	parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,
223	n/a	help=('directory to prepend to file paths for use in '
224	n/a	'compile-time tracebacks and in runtime '
225	n/a	'tracebacks in cases where the source file is '
226	n/a	'unavailable'))
227	n/a	parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
228	n/a	help=('skip files matching the regular expression; '
229	n/a	'the regexp is searched for in the full path '
230	n/a	'of each file considered for compilation'))
231	n/a	parser.add_argument('-i', metavar='FILE', dest='flist',
232	n/a	help=('add all the files and directories listed in '
233	n/a	'FILE to the list considered for compilation; '
234	n/a	'if "-", names are read from stdin'))
235	n/a	parser.add_argument('compile_dest', metavar='FILE\|DIR', nargs='*',
236	n/a	help=('zero or more file and directory names '
237	n/a	'to compile; if no arguments given, defaults '
238	n/a	'to the equivalent of -l sys.path'))
239	n/a	parser.add_argument('-j', '--workers', default=1,
240	n/a	type=int, help='Run compileall concurrently')
241	n/a
242	n/a	args = parser.parse_args()
243	n/a	compile_dests = args.compile_dest
244	n/a
245	n/a	if args.rx:
246	n/a	import re
247	n/a	args.rx = re.compile(args.rx)
248	n/a
249	n/a
250	n/a	if args.recursion is not None:
251	n/a	maxlevels = args.recursion
252	n/a	else:
253	n/a	maxlevels = args.maxlevels
254	n/a
255	n/a	# if flist is provided then load it
256	n/a	if args.flist:
257	n/a	try:
258	n/a	with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
259	n/a	for line in f:
260	n/a	compile_dests.append(line.strip())
261	n/a	except OSError:
262	n/a	if args.quiet < 2:
263	n/a	print("Error reading file list {}".format(args.flist))
264	n/a	return False
265	n/a
266	n/a	if args.workers is not None:
267	n/a	args.workers = args.workers or None
268	n/a
269	n/a	success = True
270	n/a	try:
271	n/a	if compile_dests:
272	n/a	for dest in compile_dests:
273	n/a	if os.path.isfile(dest):
274	n/a	if not compile_file(dest, args.ddir, args.force, args.rx,
275	n/a	args.quiet, args.legacy):
276	n/a	success = False
277	n/a	else:
278	n/a	if not compile_dir(dest, maxlevels, args.ddir,
279	n/a	args.force, args.rx, args.quiet,
280	n/a	args.legacy, workers=args.workers):
281	n/a	success = False
282	n/a	return success
283	n/a	else:
284	n/a	return compile_path(legacy=args.legacy, force=args.force,
285	n/a	quiet=args.quiet)
286	n/a	except KeyboardInterrupt:
287	n/a	if args.quiet < 2:
288	n/a	print("\n[interrupted]")
289	n/a	return False
290	n/a	return True
291	n/a
292	n/a
293	n/a	if __name__ == '__main__':
294	n/a	exit_status = int(not main())
295	n/a	sys.exit(exit_status)