Python code coverage for Lib/test/test_regrtest.py

#	count	content
1	n/a	"""
2	n/a	Tests of regrtest.py.
3	n/a
4	n/a	Note: test_regrtest cannot be run twice in parallel.
5	n/a	"""
6	n/a
7	n/a	import contextlib
8	n/a	import faulthandler
9	n/a	import io
10	n/a	import os.path
11	n/a	import platform
12	n/a	import re
13	n/a	import subprocess
14	n/a	import sys
15	n/a	import sysconfig
16	n/a	import tempfile
17	n/a	import textwrap
18	n/a	import unittest
19	n/a	from test import libregrtest
20	n/a	from test import support
21	n/a
22	n/a
23	n/a	Py_DEBUG = hasattr(sys, 'getobjects')
24	n/a	ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
25	n/a	ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
26	n/a
27	n/a	TEST_INTERRUPTED = textwrap.dedent("""
28	n/a	from signal import SIGINT
29	n/a	try:
30	n/a	from _testcapi import raise_signal
31	n/a	raise_signal(SIGINT)
32	n/a	except ImportError:
33	n/a	import os
34	n/a	os.kill(os.getpid(), SIGINT)
35	n/a	""")
36	n/a
37	n/a
38	n/a	class ParseArgsTestCase(unittest.TestCase):
39	n/a	"""
40	n/a	Test regrtest's argument parsing, function _parse_args().
41	n/a	"""
42	n/a
43	n/a	def checkError(self, args, msg):
44	n/a	with support.captured_stderr() as err, self.assertRaises(SystemExit):
45	n/a	libregrtest._parse_args(args)
46	n/a	self.assertIn(msg, err.getvalue())
47	n/a
48	n/a	def test_help(self):
49	n/a	for opt in '-h', '--help':
50	n/a	with self.subTest(opt=opt):
51	n/a	with support.captured_stdout() as out, \
52	n/a	self.assertRaises(SystemExit):
53	n/a	libregrtest._parse_args([opt])
54	n/a	self.assertIn('Run Python regression tests.', out.getvalue())
55	n/a
56	n/a	@unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'),
57	n/a	"faulthandler.dump_traceback_later() required")
58	n/a	def test_timeout(self):
59	n/a	ns = libregrtest._parse_args(['--timeout', '4.2'])
60	n/a	self.assertEqual(ns.timeout, 4.2)
61	n/a	self.checkError(['--timeout'], 'expected one argument')
62	n/a	self.checkError(['--timeout', 'foo'], 'invalid float value')
63	n/a
64	n/a	def test_wait(self):
65	n/a	ns = libregrtest._parse_args(['--wait'])
66	n/a	self.assertTrue(ns.wait)
67	n/a
68	n/a	def test_slaveargs(self):
69	n/a	ns = libregrtest._parse_args(['--slaveargs', '[[], {}]'])
70	n/a	self.assertEqual(ns.slaveargs, '[[], {}]')
71	n/a	self.checkError(['--slaveargs'], 'expected one argument')
72	n/a
73	n/a	def test_start(self):
74	n/a	for opt in '-S', '--start':
75	n/a	with self.subTest(opt=opt):
76	n/a	ns = libregrtest._parse_args([opt, 'foo'])
77	n/a	self.assertEqual(ns.start, 'foo')
78	n/a	self.checkError([opt], 'expected one argument')
79	n/a
80	n/a	def test_verbose(self):
81	n/a	ns = libregrtest._parse_args(['-v'])
82	n/a	self.assertEqual(ns.verbose, 1)
83	n/a	ns = libregrtest._parse_args(['-vvv'])
84	n/a	self.assertEqual(ns.verbose, 3)
85	n/a	ns = libregrtest._parse_args(['--verbose'])
86	n/a	self.assertEqual(ns.verbose, 1)
87	n/a	ns = libregrtest._parse_args(['--verbose'] * 3)
88	n/a	self.assertEqual(ns.verbose, 3)
89	n/a	ns = libregrtest._parse_args([])
90	n/a	self.assertEqual(ns.verbose, 0)
91	n/a
92	n/a	def test_verbose2(self):
93	n/a	for opt in '-w', '--verbose2':
94	n/a	with self.subTest(opt=opt):
95	n/a	ns = libregrtest._parse_args([opt])
96	n/a	self.assertTrue(ns.verbose2)
97	n/a
98	n/a	def test_verbose3(self):
99	n/a	for opt in '-W', '--verbose3':
100	n/a	with self.subTest(opt=opt):
101	n/a	ns = libregrtest._parse_args([opt])
102	n/a	self.assertTrue(ns.verbose3)
103	n/a
104	n/a	def test_quiet(self):
105	n/a	for opt in '-q', '--quiet':
106	n/a	with self.subTest(opt=opt):
107	n/a	ns = libregrtest._parse_args([opt])
108	n/a	self.assertTrue(ns.quiet)
109	n/a	self.assertEqual(ns.verbose, 0)
110	n/a
111	n/a	def test_slow(self):
112	n/a	for opt in '-o', '--slowest':
113	n/a	with self.subTest(opt=opt):
114	n/a	ns = libregrtest._parse_args([opt])
115	n/a	self.assertTrue(ns.print_slow)
116	n/a
117	n/a	def test_header(self):
118	n/a	ns = libregrtest._parse_args(['--header'])
119	n/a	self.assertTrue(ns.header)
120	n/a
121	n/a	def test_randomize(self):
122	n/a	for opt in '-r', '--randomize':
123	n/a	with self.subTest(opt=opt):
124	n/a	ns = libregrtest._parse_args([opt])
125	n/a	self.assertTrue(ns.randomize)
126	n/a
127	n/a	def test_randseed(self):
128	n/a	ns = libregrtest._parse_args(['--randseed', '12345'])
129	n/a	self.assertEqual(ns.random_seed, 12345)
130	n/a	self.assertTrue(ns.randomize)
131	n/a	self.checkError(['--randseed'], 'expected one argument')
132	n/a	self.checkError(['--randseed', 'foo'], 'invalid int value')
133	n/a
134	n/a	def test_fromfile(self):
135	n/a	for opt in '-f', '--fromfile':
136	n/a	with self.subTest(opt=opt):
137	n/a	ns = libregrtest._parse_args([opt, 'foo'])
138	n/a	self.assertEqual(ns.fromfile, 'foo')
139	n/a	self.checkError([opt], 'expected one argument')
140	n/a	self.checkError([opt, 'foo', '-s'], "don't go together")
141	n/a
142	n/a	def test_exclude(self):
143	n/a	for opt in '-x', '--exclude':
144	n/a	with self.subTest(opt=opt):
145	n/a	ns = libregrtest._parse_args([opt])
146	n/a	self.assertTrue(ns.exclude)
147	n/a
148	n/a	def test_single(self):
149	n/a	for opt in '-s', '--single':
150	n/a	with self.subTest(opt=opt):
151	n/a	ns = libregrtest._parse_args([opt])
152	n/a	self.assertTrue(ns.single)
153	n/a	self.checkError([opt, '-f', 'foo'], "don't go together")
154	n/a
155	n/a	def test_match(self):
156	n/a	for opt in '-m', '--match':
157	n/a	with self.subTest(opt=opt):
158	n/a	ns = libregrtest._parse_args([opt, 'pattern'])
159	n/a	self.assertEqual(ns.match_tests, 'pattern')
160	n/a	self.checkError([opt], 'expected one argument')
161	n/a
162	n/a	def test_failfast(self):
163	n/a	for opt in '-G', '--failfast':
164	n/a	with self.subTest(opt=opt):
165	n/a	ns = libregrtest._parse_args([opt, '-v'])
166	n/a	self.assertTrue(ns.failfast)
167	n/a	ns = libregrtest._parse_args([opt, '-W'])
168	n/a	self.assertTrue(ns.failfast)
169	n/a	self.checkError([opt], '-G/--failfast needs either -v or -W')
170	n/a
171	n/a	def test_use(self):
172	n/a	for opt in '-u', '--use':
173	n/a	with self.subTest(opt=opt):
174	n/a	ns = libregrtest._parse_args([opt, 'gui,network'])
175	n/a	self.assertEqual(ns.use_resources, ['gui', 'network'])
176	n/a	ns = libregrtest._parse_args([opt, 'gui,none,network'])
177	n/a	self.assertEqual(ns.use_resources, ['network'])
178	n/a	expected = list(libregrtest.RESOURCE_NAMES)
179	n/a	expected.remove('gui')
180	n/a	ns = libregrtest._parse_args([opt, 'all,-gui'])
181	n/a	self.assertEqual(ns.use_resources, expected)
182	n/a	self.checkError([opt], 'expected one argument')
183	n/a	self.checkError([opt, 'foo'], 'invalid resource')
184	n/a
185	n/a	def test_memlimit(self):
186	n/a	for opt in '-M', '--memlimit':
187	n/a	with self.subTest(opt=opt):
188	n/a	ns = libregrtest._parse_args([opt, '4G'])
189	n/a	self.assertEqual(ns.memlimit, '4G')
190	n/a	self.checkError([opt], 'expected one argument')
191	n/a
192	n/a	def test_testdir(self):
193	n/a	ns = libregrtest._parse_args(['--testdir', 'foo'])
194	n/a	self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo'))
195	n/a	self.checkError(['--testdir'], 'expected one argument')
196	n/a
197	n/a	def test_runleaks(self):
198	n/a	for opt in '-L', '--runleaks':
199	n/a	with self.subTest(opt=opt):
200	n/a	ns = libregrtest._parse_args([opt])
201	n/a	self.assertTrue(ns.runleaks)
202	n/a
203	n/a	def test_huntrleaks(self):
204	n/a	for opt in '-R', '--huntrleaks':
205	n/a	with self.subTest(opt=opt):
206	n/a	ns = libregrtest._parse_args([opt, ':'])
207	n/a	self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt'))
208	n/a	ns = libregrtest._parse_args([opt, '6:'])
209	n/a	self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt'))
210	n/a	ns = libregrtest._parse_args([opt, ':3'])
211	n/a	self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt'))
212	n/a	ns = libregrtest._parse_args([opt, '6:3:leaks.log'])
213	n/a	self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log'))
214	n/a	self.checkError([opt], 'expected one argument')
215	n/a	self.checkError([opt, '6'],
216	n/a	'needs 2 or 3 colon-separated arguments')
217	n/a	self.checkError([opt, 'foo:'], 'invalid huntrleaks value')
218	n/a	self.checkError([opt, '6:foo'], 'invalid huntrleaks value')
219	n/a
220	n/a	def test_multiprocess(self):
221	n/a	for opt in '-j', '--multiprocess':
222	n/a	with self.subTest(opt=opt):
223	n/a	ns = libregrtest._parse_args([opt, '2'])
224	n/a	self.assertEqual(ns.use_mp, 2)
225	n/a	self.checkError([opt], 'expected one argument')
226	n/a	self.checkError([opt, 'foo'], 'invalid int value')
227	n/a	self.checkError([opt, '2', '-T'], "don't go together")
228	n/a	self.checkError([opt, '2', '-l'], "don't go together")
229	n/a	self.checkError([opt, '0', '-T'], "don't go together")
230	n/a	self.checkError([opt, '0', '-l'], "don't go together")
231	n/a
232	n/a	def test_coverage(self):
233	n/a	for opt in '-T', '--coverage':
234	n/a	with self.subTest(opt=opt):
235	n/a	ns = libregrtest._parse_args([opt])
236	n/a	self.assertTrue(ns.trace)
237	n/a
238	n/a	def test_coverdir(self):
239	n/a	for opt in '-D', '--coverdir':
240	n/a	with self.subTest(opt=opt):
241	n/a	ns = libregrtest._parse_args([opt, 'foo'])
242	n/a	self.assertEqual(ns.coverdir,
243	n/a	os.path.join(support.SAVEDCWD, 'foo'))
244	n/a	self.checkError([opt], 'expected one argument')
245	n/a
246	n/a	def test_nocoverdir(self):
247	n/a	for opt in '-N', '--nocoverdir':
248	n/a	with self.subTest(opt=opt):
249	n/a	ns = libregrtest._parse_args([opt])
250	n/a	self.assertIsNone(ns.coverdir)
251	n/a
252	n/a	def test_threshold(self):
253	n/a	for opt in '-t', '--threshold':
254	n/a	with self.subTest(opt=opt):
255	n/a	ns = libregrtest._parse_args([opt, '1000'])
256	n/a	self.assertEqual(ns.threshold, 1000)
257	n/a	self.checkError([opt], 'expected one argument')
258	n/a	self.checkError([opt, 'foo'], 'invalid int value')
259	n/a
260	n/a	def test_nowindows(self):
261	n/a	for opt in '-n', '--nowindows':
262	n/a	with self.subTest(opt=opt):
263	n/a	with contextlib.redirect_stderr(io.StringIO()) as stderr:
264	n/a	ns = libregrtest._parse_args([opt])
265	n/a	self.assertTrue(ns.nowindows)
266	n/a	err = stderr.getvalue()
267	n/a	self.assertIn('the --nowindows (-n) option is deprecated', err)
268	n/a
269	n/a	def test_forever(self):
270	n/a	for opt in '-F', '--forever':
271	n/a	with self.subTest(opt=opt):
272	n/a	ns = libregrtest._parse_args([opt])
273	n/a	self.assertTrue(ns.forever)
274	n/a
275	n/a
276	n/a	def test_unrecognized_argument(self):
277	n/a	self.checkError(['--xxx'], 'usage:')
278	n/a
279	n/a	def test_long_option__partial(self):
280	n/a	ns = libregrtest._parse_args(['--qui'])
281	n/a	self.assertTrue(ns.quiet)
282	n/a	self.assertEqual(ns.verbose, 0)
283	n/a
284	n/a	def test_two_options(self):
285	n/a	ns = libregrtest._parse_args(['--quiet', '--exclude'])
286	n/a	self.assertTrue(ns.quiet)
287	n/a	self.assertEqual(ns.verbose, 0)
288	n/a	self.assertTrue(ns.exclude)
289	n/a
290	n/a	def test_option_with_empty_string_value(self):
291	n/a	ns = libregrtest._parse_args(['--start', ''])
292	n/a	self.assertEqual(ns.start, '')
293	n/a
294	n/a	def test_arg(self):
295	n/a	ns = libregrtest._parse_args(['foo'])
296	n/a	self.assertEqual(ns.args, ['foo'])
297	n/a
298	n/a	def test_option_and_arg(self):
299	n/a	ns = libregrtest._parse_args(['--quiet', 'foo'])
300	n/a	self.assertTrue(ns.quiet)
301	n/a	self.assertEqual(ns.verbose, 0)
302	n/a	self.assertEqual(ns.args, ['foo'])
303	n/a
304	n/a	def test_arg_option_arg(self):
305	n/a	ns = libregrtest._parse_args(['test_unaryop', '-v', 'test_binop'])
306	n/a	self.assertEqual(ns.verbose, 1)
307	n/a	self.assertEqual(ns.args, ['test_unaryop', 'test_binop'])
308	n/a
309	n/a	def test_unknown_option(self):
310	n/a	self.checkError(['--unknown-option'],
311	n/a	'unrecognized arguments: --unknown-option')
312	n/a
313	n/a
314	n/a	class BaseTestCase(unittest.TestCase):
315	n/a	TEST_UNIQUE_ID = 1
316	n/a	TESTNAME_PREFIX = 'test_regrtest_'
317	n/a	TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+'
318	n/a
319	n/a	def setUp(self):
320	n/a	self.testdir = os.path.realpath(os.path.dirname(__file__))
321	n/a
322	n/a	self.tmptestdir = tempfile.mkdtemp()
323	n/a	self.addCleanup(support.rmtree, self.tmptestdir)
324	n/a
325	n/a	def create_test(self, name=None, code=''):
326	n/a	if not name:
327	n/a	name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID
328	n/a	BaseTestCase.TEST_UNIQUE_ID += 1
329	n/a
330	n/a	# test_regrtest cannot be run twice in parallel because
331	n/a	# of setUp() and create_test()
332	n/a	name = self.TESTNAME_PREFIX + name
333	n/a	path = os.path.join(self.tmptestdir, name + '.py')
334	n/a
335	n/a	self.addCleanup(support.unlink, path)
336	n/a	# Use 'x' mode to ensure that we do not override existing tests
337	n/a	try:
338	n/a	with open(path, 'x', encoding='utf-8') as fp:
339	n/a	fp.write(code)
340	n/a	except PermissionError as exc:
341	n/a	if not sysconfig.is_python_build():
342	n/a	self.skipTest("cannot write %s: %s" % (path, exc))
343	n/a	raise
344	n/a	return name
345	n/a
346	n/a	def regex_search(self, regex, output):
347	n/a	match = re.search(regex, output, re.MULTILINE)
348	n/a	if not match:
349	n/a	self.fail("%r not found in %r" % (regex, output))
350	n/a	return match
351	n/a
352	n/a	def check_line(self, output, regex):
353	n/a	regex = re.compile(r'^' + regex, re.MULTILINE)
354	n/a	self.assertRegex(output, regex)
355	n/a
356	n/a	def parse_executed_tests(self, output):
357	n/a	regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ [0-9]+(?:/ [0-9]+)*\] (%s)'
358	n/a	% self.TESTNAME_REGEX)
359	n/a	parser = re.finditer(regex, output, re.MULTILINE)
360	n/a	return list(match.group(1) for match in parser)
361	n/a
362	n/a	def check_executed_tests(self, output, tests, skipped=(), failed=(),
363	n/a	omitted=(), randomize=False, interrupted=False):
364	n/a	if isinstance(tests, str):
365	n/a	tests = [tests]
366	n/a	if isinstance(skipped, str):
367	n/a	skipped = [skipped]
368	n/a	if isinstance(failed, str):
369	n/a	failed = [failed]
370	n/a	if isinstance(omitted, str):
371	n/a	omitted = [omitted]
372	n/a	ntest = len(tests)
373	n/a	nskipped = len(skipped)
374	n/a	nfailed = len(failed)
375	n/a	nomitted = len(omitted)
376	n/a
377	n/a	executed = self.parse_executed_tests(output)
378	n/a	if randomize:
379	n/a	self.assertEqual(set(executed), set(tests), output)
380	n/a	else:
381	n/a	self.assertEqual(executed, tests, output)
382	n/a
383	n/a	def plural(count):
384	n/a	return 's' if count != 1 else ''
385	n/a
386	n/a	def list_regex(line_format, tests):
387	n/a	count = len(tests)
388	n/a	names = ' '.join(sorted(tests))
389	n/a	regex = line_format % (count, plural(count))
390	n/a	regex = r'%s:\n %s$' % (regex, names)
391	n/a	return regex
392	n/a
393	n/a	if skipped:
394	n/a	regex = list_regex('%s test%s skipped', skipped)
395	n/a	self.check_line(output, regex)
396	n/a
397	n/a	if failed:
398	n/a	regex = list_regex('%s test%s failed', failed)
399	n/a	self.check_line(output, regex)
400	n/a
401	n/a	if omitted:
402	n/a	regex = list_regex('%s test%s omitted', omitted)
403	n/a	self.check_line(output, regex)
404	n/a
405	n/a	good = ntest - nskipped - nfailed - nomitted
406	n/a	if good:
407	n/a	regex = r'%s test%s OK\.$' % (good, plural(good))
408	n/a	if not skipped and not failed and good > 1:
409	n/a	regex = 'All %s' % regex
410	n/a	self.check_line(output, regex)
411	n/a
412	n/a	if interrupted:
413	n/a	self.check_line(output, 'Test suite interrupted by signal SIGINT.')
414	n/a
415	n/a	if nfailed:
416	n/a	result = 'FAILURE'
417	n/a	elif interrupted:
418	n/a	result = 'INTERRUPTED'
419	n/a	else:
420	n/a	result = 'SUCCESS'
421	n/a	self.check_line(output, 'Tests result: %s' % result)
422	n/a
423	n/a	def parse_random_seed(self, output):
424	n/a	match = self.regex_search(r'Using random seed ([0-9]+)', output)
425	n/a	randseed = int(match.group(1))
426	n/a	self.assertTrue(0 <= randseed <= 10000000, randseed)
427	n/a	return randseed
428	n/a
429	n/a	def run_command(self, args, input=None, exitcode=0, **kw):
430	n/a	if not input:
431	n/a	input = ''
432	n/a	if 'stderr' not in kw:
433	n/a	kw['stderr'] = subprocess.PIPE
434	n/a	proc = subprocess.run(args,
435	n/a	universal_newlines=True,
436	n/a	input=input,
437	n/a	stdout=subprocess.PIPE,
438	n/a	**kw)
439	n/a	if proc.returncode != exitcode:
440	n/a	msg = ("Command %s failed with exit code %s\n"
441	n/a	"\n"
442	n/a	"stdout:\n"
443	n/a	"---\n"
444	n/a	"%s\n"
445	n/a	"---\n"
446	n/a	% (str(args), proc.returncode, proc.stdout))
447	n/a	if proc.stderr:
448	n/a	msg += ("\n"
449	n/a	"stderr:\n"
450	n/a	"---\n"
451	n/a	"%s"
452	n/a	"---\n"
453	n/a	% proc.stderr)
454	n/a	self.fail(msg)
455	n/a	return proc
456	n/a
457	n/a
458	n/a	def run_python(self, args, **kw):
459	n/a	args = [sys.executable, '-X', 'faulthandler', '-I', *args]
460	n/a	proc = self.run_command(args, **kw)
461	n/a	return proc.stdout
462	n/a
463	n/a
464	n/a	class ProgramsTestCase(BaseTestCase):
465	n/a	"""
466	n/a	Test various ways to run the Python test suite. Use options close
467	n/a	to options used on the buildbot.
468	n/a	"""
469	n/a
470	n/a	NTEST = 4
471	n/a
472	n/a	def setUp(self):
473	n/a	super().setUp()
474	n/a
475	n/a	# Create NTEST tests doing nothing
476	n/a	self.tests = [self.create_test() for index in range(self.NTEST)]
477	n/a
478	n/a	self.python_args = ['-Wd', '-E', '-bb']
479	n/a	self.regrtest_args = ['-uall', '-rwW',
480	n/a	'--testdir=%s' % self.tmptestdir]
481	n/a	if hasattr(faulthandler, 'dump_traceback_later'):
482	n/a	self.regrtest_args.extend(('--timeout', '3600', '-j4'))
483	n/a	if sys.platform == 'win32':
484	n/a	self.regrtest_args.append('-n')
485	n/a
486	n/a	def check_output(self, output):
487	n/a	self.parse_random_seed(output)
488	n/a	self.check_executed_tests(output, self.tests, randomize=True)
489	n/a
490	n/a	def run_tests(self, args):
491	n/a	output = self.run_python(args)
492	n/a	self.check_output(output)
493	n/a
494	n/a	def test_script_regrtest(self):
495	n/a	# Lib/test/regrtest.py
496	n/a	script = os.path.join(self.testdir, 'regrtest.py')
497	n/a
498	n/a	args = [self.python_args, script, self.regrtest_args, *self.tests]
499	n/a	self.run_tests(args)
500	n/a
501	n/a	def test_module_test(self):
502	n/a	# -m test
503	n/a	args = [*self.python_args, '-m', 'test',
504	n/a	self.regrtest_args, self.tests]
505	n/a	self.run_tests(args)
506	n/a
507	n/a	def test_module_regrtest(self):
508	n/a	# -m test.regrtest
509	n/a	args = [*self.python_args, '-m', 'test.regrtest',
510	n/a	self.regrtest_args, self.tests]
511	n/a	self.run_tests(args)
512	n/a
513	n/a	def test_module_autotest(self):
514	n/a	# -m test.autotest
515	n/a	args = [*self.python_args, '-m', 'test.autotest',
516	n/a	self.regrtest_args, self.tests]
517	n/a	self.run_tests(args)
518	n/a
519	n/a	def test_module_from_test_autotest(self):
520	n/a	# from test import autotest
521	n/a	code = 'from test import autotest'
522	n/a	args = [*self.python_args, '-c', code,
523	n/a	self.regrtest_args, self.tests]
524	n/a	self.run_tests(args)
525	n/a
526	n/a	def test_script_autotest(self):
527	n/a	# Lib/test/autotest.py
528	n/a	script = os.path.join(self.testdir, 'autotest.py')
529	n/a	args = [self.python_args, script, self.regrtest_args, *self.tests]
530	n/a	self.run_tests(args)
531	n/a
532	n/a	@unittest.skipUnless(sysconfig.is_python_build(),
533	n/a	'run_tests.py script is not installed')
534	n/a	def test_tools_script_run_tests(self):
535	n/a	# Tools/scripts/run_tests.py
536	n/a	script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
537	n/a	args = [script, self.regrtest_args, self.tests]
538	n/a	self.run_tests(args)
539	n/a
540	n/a	def run_batch(self, *args):
541	n/a	proc = self.run_command(args)
542	n/a	self.check_output(proc.stdout)
543	n/a
544	n/a	@unittest.skipUnless(sysconfig.is_python_build(),
545	n/a	'test.bat script is not installed')
546	n/a	@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
547	n/a	def test_tools_buildbot_test(self):
548	n/a	# Tools\buildbot\test.bat
549	n/a	script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat')
550	n/a	test_args = ['--testdir=%s' % self.tmptestdir]
551	n/a	if platform.architecture()[0] == '64bit':
552	n/a	test_args.append('-x64') # 64-bit build
553	n/a	if not Py_DEBUG:
554	n/a	test_args.append('+d') # Release build, use python.exe
555	n/a	self.run_batch(script, test_args, self.tests)
556	n/a
557	n/a	@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
558	n/a	def test_pcbuild_rt(self):
559	n/a	# PCbuild\rt.bat
560	n/a	script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat')
561	n/a	rt_args = ["-q"] # Quick, don't run tests twice
562	n/a	if platform.architecture()[0] == '64bit':
563	n/a	rt_args.append('-x64') # 64-bit build
564	n/a	if Py_DEBUG:
565	n/a	rt_args.append('-d') # Debug build, use python_d.exe
566	n/a	self.run_batch(script, rt_args, self.regrtest_args, *self.tests)
567	n/a
568	n/a
569	n/a	class ArgsTestCase(BaseTestCase):
570	n/a	"""
571	n/a	Test arguments of the Python test suite.
572	n/a	"""
573	n/a
574	n/a	def run_tests(self, testargs, *kw):
575	n/a	cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs]
576	n/a	return self.run_python(cmdargs, **kw)
577	n/a
578	n/a	def test_failing_test(self):
579	n/a	# test a failing test
580	n/a	code = textwrap.dedent("""
581	n/a	import unittest
582	n/a
583	n/a	class FailingTest(unittest.TestCase):
584	n/a	def test_failing(self):
585	n/a	self.fail("bug")
586	n/a	""")
587	n/a	test_ok = self.create_test('ok')
588	n/a	test_failing = self.create_test('failing', code=code)
589	n/a	tests = [test_ok, test_failing]
590	n/a
591	n/a	output = self.run_tests(*tests, exitcode=1)
592	n/a	self.check_executed_tests(output, tests, failed=test_failing)
593	n/a
594	n/a	def test_resources(self):
595	n/a	# test -u command line option
596	n/a	tests = {}
597	n/a	for resource in ('audio', 'network'):
598	n/a	code = 'from test import support\nsupport.requires(%r)' % resource
599	n/a	tests[resource] = self.create_test(resource, code)
600	n/a	test_names = sorted(tests.values())
601	n/a
602	n/a	# -u all: 2 resources enabled
603	n/a	output = self.run_tests('-u', 'all', *test_names)
604	n/a	self.check_executed_tests(output, test_names)
605	n/a
606	n/a	# -u audio: 1 resource enabled
607	n/a	output = self.run_tests('-uaudio', *test_names)
608	n/a	self.check_executed_tests(output, test_names,
609	n/a	skipped=tests['network'])
610	n/a
611	n/a	# no option: 0 resources enabled
612	n/a	output = self.run_tests(*test_names)
613	n/a	self.check_executed_tests(output, test_names,
614	n/a	skipped=test_names)
615	n/a
616	n/a	def test_random(self):
617	n/a	# test -r and --randseed command line option
618	n/a	code = textwrap.dedent("""
619	n/a	import random
620	n/a	print("TESTRANDOM: %s" % random.randint(1, 1000))
621	n/a	""")
622	n/a	test = self.create_test('random', code)
623	n/a
624	n/a	# first run to get the output with the random seed
625	n/a	output = self.run_tests('-r', test)
626	n/a	randseed = self.parse_random_seed(output)
627	n/a	match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
628	n/a	test_random = int(match.group(1))
629	n/a
630	n/a	# try to reproduce with the random seed
631	n/a	output = self.run_tests('-r', '--randseed=%s' % randseed, test)
632	n/a	randseed2 = self.parse_random_seed(output)
633	n/a	self.assertEqual(randseed2, randseed)
634	n/a
635	n/a	match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
636	n/a	test_random2 = int(match.group(1))
637	n/a	self.assertEqual(test_random2, test_random)
638	n/a
639	n/a	def test_fromfile(self):
640	n/a	# test --fromfile
641	n/a	tests = [self.create_test() for index in range(5)]
642	n/a
643	n/a	# Write the list of files using a format similar to regrtest output:
644	n/a	# [1/2] test_1
645	n/a	# [2/2] test_2
646	n/a	filename = support.TESTFN
647	n/a	self.addCleanup(support.unlink, filename)
648	n/a
649	n/a	# test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec'
650	n/a	with open(filename, "w") as fp:
651	n/a	previous = None
652	n/a	for index, name in enumerate(tests, 1):
653	n/a	line = ("00:00:%02i [%s/%s] %s"
654	n/a	% (index, index, len(tests), name))
655	n/a	if previous:
656	n/a	line += " -- %s took 0 sec" % previous
657	n/a	print(line, file=fp)
658	n/a	previous = name
659	n/a
660	n/a	output = self.run_tests('--fromfile', filename)
661	n/a	self.check_executed_tests(output, tests)
662	n/a
663	n/a	# test format '[2/7] test_opcodes'
664	n/a	with open(filename, "w") as fp:
665	n/a	for index, name in enumerate(tests, 1):
666	n/a	print("[%s/%s] %s" % (index, len(tests), name), file=fp)
667	n/a
668	n/a	output = self.run_tests('--fromfile', filename)
669	n/a	self.check_executed_tests(output, tests)
670	n/a
671	n/a	# test format 'test_opcodes'
672	n/a	with open(filename, "w") as fp:
673	n/a	for name in tests:
674	n/a	print(name, file=fp)
675	n/a
676	n/a	output = self.run_tests('--fromfile', filename)
677	n/a	self.check_executed_tests(output, tests)
678	n/a
679	n/a	# test format 'Lib/test/test_opcodes.py'
680	n/a	with open(filename, "w") as fp:
681	n/a	for name in tests:
682	n/a	print('Lib/test/%s.py' % name, file=fp)
683	n/a
684	n/a	output = self.run_tests('--fromfile', filename)
685	n/a	self.check_executed_tests(output, tests)
686	n/a
687	n/a	def test_interrupted(self):
688	n/a	code = TEST_INTERRUPTED
689	n/a	test = self.create_test('sigint', code=code)
690	n/a	output = self.run_tests(test, exitcode=1)
691	n/a	self.check_executed_tests(output, test, omitted=test,
692	n/a	interrupted=True)
693	n/a
694	n/a	def test_slowest(self):
695	n/a	# test --slowest
696	n/a	tests = [self.create_test() for index in range(3)]
697	n/a	output = self.run_tests("--slowest", *tests)
698	n/a	self.check_executed_tests(output, tests)
699	n/a	regex = ('10 slowest tests:\n'
700	n/a	'(?:- %s: .*\n){%s}'
701	n/a	% (self.TESTNAME_REGEX, len(tests)))
702	n/a	self.check_line(output, regex)
703	n/a
704	n/a	def test_slow_interrupted(self):
705	n/a	# Issue #25373: test --slowest with an interrupted test
706	n/a	code = TEST_INTERRUPTED
707	n/a	test = self.create_test("sigint", code=code)
708	n/a
709	n/a	try:
710	n/a	import threading
711	n/a	tests = (False, True)
712	n/a	except ImportError:
713	n/a	tests = (False,)
714	n/a	for multiprocessing in tests:
715	n/a	if multiprocessing:
716	n/a	args = ("--slowest", "-j2", test)
717	n/a	else:
718	n/a	args = ("--slowest", test)
719	n/a	output = self.run_tests(*args, exitcode=1)
720	n/a	self.check_executed_tests(output, test,
721	n/a	omitted=test, interrupted=True)
722	n/a
723	n/a	regex = ('10 slowest tests:\n')
724	n/a	self.check_line(output, regex)
725	n/a
726	n/a	def test_coverage(self):
727	n/a	# test --coverage
728	n/a	test = self.create_test('coverage')
729	n/a	output = self.run_tests("--coverage", test)
730	n/a	self.check_executed_tests(output, [test])
731	n/a	regex = (r'lines +cov% +module +$path$\n'
732	n/a	r'(?: [0-9]+ [0-9]{1,2}% *[^ ]+ +$[^)]+$+)+')
733	n/a	self.check_line(output, regex)
734	n/a
735	n/a	def test_wait(self):
736	n/a	# test --wait
737	n/a	test = self.create_test('wait')
738	n/a	output = self.run_tests("--wait", test, input='key')
739	n/a	self.check_line(output, 'Press any key to continue')
740	n/a
741	n/a	def test_forever(self):
742	n/a	# test --forever
743	n/a	code = textwrap.dedent("""
744	n/a	import builtins
745	n/a	import unittest
746	n/a
747	n/a	class ForeverTester(unittest.TestCase):
748	n/a	def test_run(self):
749	n/a	# Store the state in the builtins module, because the test
750	n/a	# module is reload at each run
751	n/a	if 'RUN' in builtins.__dict__:
752	n/a	builtins.__dict__['RUN'] += 1
753	n/a	if builtins.__dict__['RUN'] >= 3:
754	n/a	self.fail("fail at the 3rd runs")
755	n/a	else:
756	n/a	builtins.__dict__['RUN'] = 1
757	n/a	""")
758	n/a	test = self.create_test('forever', code=code)
759	n/a	output = self.run_tests('--forever', test, exitcode=1)
760	n/a	self.check_executed_tests(output, [test]*3, failed=test)
761	n/a
762	n/a	@unittest.skipUnless(Py_DEBUG, 'need a debug build')
763	n/a	def test_huntrleaks_fd_leak(self):
764	n/a	# test --huntrleaks for file descriptor leak
765	n/a	code = textwrap.dedent("""
766	n/a	import os
767	n/a	import unittest
768	n/a
769	n/a	# Issue #25306: Disable popups and logs to stderr on assertion
770	n/a	# failures in MSCRT
771	n/a	try:
772	n/a	import msvcrt
773	n/a	msvcrt.CrtSetReportMode
774	n/a	except (ImportError, AttributeError):
775	n/a	# no Windows, o release build
776	n/a	pass
777	n/a	else:
778	n/a	for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
779	n/a	msvcrt.CrtSetReportMode(m, 0)
780	n/a
781	n/a	class FDLeakTest(unittest.TestCase):
782	n/a	def test_leak(self):
783	n/a	fd = os.open(__file__, os.O_RDONLY)
784	n/a	# bug: never cloes the file descriptor
785	n/a	""")
786	n/a	test = self.create_test('huntrleaks', code=code)
787	n/a
788	n/a	filename = 'reflog.txt'
789	n/a	self.addCleanup(support.unlink, filename)
790	n/a	output = self.run_tests('--huntrleaks', '3:3:', test,
791	n/a	exitcode=1,
792	n/a	stderr=subprocess.STDOUT)
793	n/a	self.check_executed_tests(output, [test], failed=test)
794	n/a
795	n/a	line = 'beginning 6 repetitions\n123456\n......\n'
796	n/a	self.check_line(output, re.escape(line))
797	n/a
798	n/a	line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test
799	n/a	self.assertIn(line2, output)
800	n/a
801	n/a	with open(filename) as fp:
802	n/a	reflog = fp.read()
803	n/a	self.assertIn(line2, reflog)
804	n/a
805	n/a	def test_list_tests(self):
806	n/a	# test --list-tests
807	n/a	tests = [self.create_test() for i in range(5)]
808	n/a	output = self.run_tests('--list-tests', *tests)
809	n/a	self.assertEqual(output.rstrip().splitlines(),
810	n/a	tests)
811	n/a
812	n/a	def test_crashed(self):
813	n/a	# Any code which causes a crash
814	n/a	code = 'import faulthandler; faulthandler._sigsegv()'
815	n/a	crash_test = self.create_test(name="crash", code=code)
816	n/a	ok_test = self.create_test(name="ok")
817	n/a
818	n/a	tests = [crash_test, ok_test]
819	n/a	output = self.run_tests("-j2", *tests, exitcode=1)
820	n/a	self.check_executed_tests(output, tests, failed=crash_test,
821	n/a	randomize=True)
822	n/a
823	n/a
824	n/a	if __name__ == '__main__':
825	n/a	unittest.main()