Python code coverage for Lib/importlib/test/benchmark.py

#	count	content
1	n/a	"""Benchmark some basic import use-cases.
2	n/a
3	n/a	The assumption is made that this benchmark is run in a fresh interpreter and
4	n/a	thus has no external changes made to import-related attributes in sys.
5	n/a
6	n/a	"""
7	n/a	from . import util
8	n/a	from .source import util as source_util
9	n/a	import decimal
10	n/a	import imp
11	n/a	import importlib
12	n/a	import importlib.machinery
13	n/a	import json
14	n/a	import os
15	n/a	import py_compile
16	n/a	import sys
17	n/a	import tabnanny
18	n/a	import timeit
19	n/a
20	n/a
21	n/a	def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3):
22	n/a	"""Bench the given statement as many times as necessary until total
23	n/a	executions take one second."""
24	n/a	stmt = "__import__({!r})".format(name)
25	n/a	timer = timeit.Timer(stmt)
26	n/a	for x in range(repeat):
27	n/a	total_time = 0
28	n/a	count = 0
29	n/a	while total_time < seconds:
30	n/a	try:
31	n/a	total_time += timer.timeit(1)
32	n/a	finally:
33	n/a	cleanup()
34	n/a	count += 1
35	n/a	else:
36	n/a	# One execution too far
37	n/a	if total_time > seconds:
38	n/a	count -= 1
39	n/a	yield count // seconds
40	n/a
41	n/a	def from_cache(seconds, repeat):
42	n/a	"""sys.modules"""
43	n/a	name = '<benchmark import>'
44	n/a	module = imp.new_module(name)
45	n/a	module.__file__ = '<test>'
46	n/a	module.__package__ = ''
47	n/a	with util.uncache(name):
48	n/a	sys.modules[name] = module
49	n/a	for result in bench(name, repeat=repeat, seconds=seconds):
50	n/a	yield result
51	n/a
52	n/a
53	n/a	def builtin_mod(seconds, repeat):
54	n/a	"""Built-in module"""
55	n/a	name = 'errno'
56	n/a	if name in sys.modules:
57	n/a	del sys.modules[name]
58	n/a	# Relying on built-in importer being implicit.
59	n/a	for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
60	n/a	seconds=seconds):
61	n/a	yield result
62	n/a
63	n/a
64	n/a	def source_wo_bytecode(seconds, repeat):
65	n/a	"""Source w/o bytecode: small"""
66	n/a	sys.dont_write_bytecode = True
67	n/a	try:
68	n/a	name = '__importlib_test_benchmark__'
69	n/a	# Clears out sys.modules and puts an entry at the front of sys.path.
70	n/a	with source_util.create_modules(name) as mapping:
71	n/a	assert not os.path.exists(imp.cache_from_source(mapping[name]))
72	n/a	sys.meta_path.append(importlib.machinery.PathFinder)
73	n/a	loader = (importlib.machinery.SourceFileLoader,
74	n/a	importlib.machinery.SOURCE_SUFFIXES, True)
75	n/a	sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader))
76	n/a	for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
77	n/a	seconds=seconds):
78	n/a	yield result
79	n/a	finally:
80	n/a	sys.dont_write_bytecode = False
81	n/a
82	n/a
83	n/a	def _wo_bytecode(module):
84	n/a	name = module.__name__
85	n/a	def benchmark_wo_bytecode(seconds, repeat):
86	n/a	"""Source w/o bytecode: {}"""
87	n/a	bytecode_path = imp.cache_from_source(module.__file__)
88	n/a	if os.path.exists(bytecode_path):
89	n/a	os.unlink(bytecode_path)
90	n/a	sys.dont_write_bytecode = True
91	n/a	try:
92	n/a	for result in bench(name, lambda: sys.modules.pop(name),
93	n/a	repeat=repeat, seconds=seconds):
94	n/a	yield result
95	n/a	finally:
96	n/a	sys.dont_write_bytecode = False
97	n/a
98	n/a	benchmark_wo_bytecode.__doc__ = benchmark_wo_bytecode.__doc__.format(name)
99	n/a	return benchmark_wo_bytecode
100	n/a
101	n/a	tabnanny_wo_bytecode = _wo_bytecode(tabnanny)
102	n/a	decimal_wo_bytecode = _wo_bytecode(decimal)
103	n/a
104	n/a
105	n/a	def source_writing_bytecode(seconds, repeat):
106	n/a	"""Source writing bytecode: small"""
107	n/a	assert not sys.dont_write_bytecode
108	n/a	name = '__importlib_test_benchmark__'
109	n/a	with source_util.create_modules(name) as mapping:
110	n/a	sys.meta_path.append(importlib.machinery.PathFinder)
111	n/a	loader = (importlib.machinery.SourceFileLoader,
112	n/a	importlib.machinery.SOURCE_SUFFIXES, True)
113	n/a	sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader))
114	n/a	def cleanup():
115	n/a	sys.modules.pop(name)
116	n/a	os.unlink(imp.cache_from_source(mapping[name]))
117	n/a	for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
118	n/a	assert not os.path.exists(imp.cache_from_source(mapping[name]))
119	n/a	yield result
120	n/a
121	n/a
122	n/a	def _writing_bytecode(module):
123	n/a	name = module.__name__
124	n/a	def writing_bytecode_benchmark(seconds, repeat):
125	n/a	"""Source writing bytecode: {}"""
126	n/a	assert not sys.dont_write_bytecode
127	n/a	def cleanup():
128	n/a	sys.modules.pop(name)
129	n/a	os.unlink(imp.cache_from_source(module.__file__))
130	n/a	for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
131	n/a	yield result
132	n/a
133	n/a	writing_bytecode_benchmark.__doc__ = (
134	n/a	writing_bytecode_benchmark.__doc__.format(name))
135	n/a	return writing_bytecode_benchmark
136	n/a
137	n/a	tabnanny_writing_bytecode = _writing_bytecode(tabnanny)
138	n/a	decimal_writing_bytecode = _writing_bytecode(decimal)
139	n/a
140	n/a
141	n/a	def source_using_bytecode(seconds, repeat):
142	n/a	"""Source w/ bytecode: small"""
143	n/a	name = '__importlib_test_benchmark__'
144	n/a	with source_util.create_modules(name) as mapping:
145	n/a	sys.meta_path.append(importlib.machinery.PathFinder)
146	n/a	loader = (importlib.machinery.SourceFileLoader,
147	n/a	importlib.machinery.SOURCE_SUFFIXES, True)
148	n/a	sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader))
149	n/a	py_compile.compile(mapping[name])
150	n/a	assert os.path.exists(imp.cache_from_source(mapping[name]))
151	n/a	for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
152	n/a	seconds=seconds):
153	n/a	yield result
154	n/a
155	n/a
156	n/a	def _using_bytecode(module):
157	n/a	name = module.__name__
158	n/a	def using_bytecode_benchmark(seconds, repeat):
159	n/a	"""Source w/ bytecode: {}"""
160	n/a	py_compile.compile(module.__file__)
161	n/a	for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
162	n/a	seconds=seconds):
163	n/a	yield result
164	n/a
165	n/a	using_bytecode_benchmark.__doc__ = (
166	n/a	using_bytecode_benchmark.__doc__.format(name))
167	n/a	return using_bytecode_benchmark
168	n/a
169	n/a	tabnanny_using_bytecode = _using_bytecode(tabnanny)
170	n/a	decimal_using_bytecode = _using_bytecode(decimal)
171	n/a
172	n/a
173	n/a	def main(import_, options):
174	n/a	if options.source_file:
175	n/a	with options.source_file:
176	n/a	prev_results = json.load(options.source_file)
177	n/a	else:
178	n/a	prev_results = {}
179	n/a	__builtins__.__import__ = import_
180	n/a	benchmarks = (from_cache, builtin_mod,
181	n/a	source_writing_bytecode,
182	n/a	source_wo_bytecode, source_using_bytecode,
183	n/a	tabnanny_writing_bytecode,
184	n/a	tabnanny_wo_bytecode, tabnanny_using_bytecode,
185	n/a	decimal_writing_bytecode,
186	n/a	decimal_wo_bytecode, decimal_using_bytecode,
187	n/a	)
188	n/a	if options.benchmark:
189	n/a	for b in benchmarks:
190	n/a	if b.__doc__ == options.benchmark:
191	n/a	benchmarks = [b]
192	n/a	break
193	n/a	else:
194	n/a	print('Unknown benchmark: {!r}'.format(options.benchmark,
195	n/a	file=sys.stderr))
196	n/a	sys.exit(1)
197	n/a	seconds = 1
198	n/a	seconds_plural = 's' if seconds > 1 else ''
199	n/a	repeat = 3
200	n/a	header = ('Measuring imports/second over {} second{}, best out of {}\n'
201	n/a	'Entire benchmark run should take about {} seconds\n'
202	n/a	'Using {!r} as __import__\n')
203	n/a	print(header.format(seconds, seconds_plural, repeat,
204	n/a	len(benchmarks) * seconds * repeat, __import__))
205	n/a	new_results = {}
206	n/a	for benchmark in benchmarks:
207	n/a	print(benchmark.__doc__, "[", end=' ')
208	n/a	sys.stdout.flush()
209	n/a	results = []
210	n/a	for result in benchmark(seconds=seconds, repeat=repeat):
211	n/a	results.append(result)
212	n/a	print(result, end=' ')
213	n/a	sys.stdout.flush()
214	n/a	assert not sys.dont_write_bytecode
215	n/a	print("]", "best is", format(max(results), ',d'))
216	n/a	new_results[benchmark.__doc__] = results
217	n/a	if prev_results:
218	n/a	print('\n\nComparing new vs. old\n')
219	n/a	for benchmark in benchmarks:
220	n/a	benchmark_name = benchmark.__doc__
221	n/a	old_result = max(prev_results[benchmark_name])
222	n/a	new_result = max(new_results[benchmark_name])
223	n/a	result = '{:,d} vs. {:,d} ({:%})'.format(new_result,
224	n/a	old_result,
225	n/a	new_result/old_result)
226	n/a	print(benchmark_name, ':', result)
227	n/a	if options.dest_file:
228	n/a	with options.dest_file:
229	n/a	json.dump(new_results, options.dest_file, indent=2)
230	n/a
231	n/a
232	n/a	if __name__ == '__main__':
233	n/a	import argparse
234	n/a
235	n/a	parser = argparse.ArgumentParser()
236	n/a	parser.add_argument('-b', '--builtin', dest='builtin', action='store_true',
237	n/a	default=False, help="use the built-in __import__")
238	n/a	parser.add_argument('-r', '--read', dest='source_file',
239	n/a	type=argparse.FileType('r'),
240	n/a	help='file to read benchmark data from to compare '
241	n/a	'against')
242	n/a	parser.add_argument('-w', '--write', dest='dest_file',
243	n/a	type=argparse.FileType('w'),
244	n/a	help='file to write benchmark data to')
245	n/a	parser.add_argument('--benchmark', dest='benchmark',
246	n/a	help='specific benchmark to run')
247	n/a	options = parser.parse_args()
248	n/a	import_ = __import__
249	n/a	if not options.builtin:
250	n/a	import_ = importlib.__import__
251	n/a
252	n/a	main(import_, options)